sys/dev/cxgb/cxgb_main.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2007-2008, Chelsio Inc.
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Neither the name of the Chelsio Corporation nor the names of its
  13     contributors may be used to endorse or promote products derived from
  14     this software without specific prior written permission.
  15
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26 POSSIBILITY OF SUCH DAMAGE.
  27
  28 ***************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/ktr.h>
  44 #include <sys/rman.h>
  45 #include <sys/ioccom.h>
  46 #include <sys/mbuf.h>
  47 #include <sys/linker.h>
  48 #include <sys/firmware.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54 #include <sys/queue.h>
  55 #include <sys/taskqueue.h>
  56 #include <sys/proc.h>
  57
  58 #include <net/bpf.h>
  59 #include <net/ethernet.h>
  60 #include <net/if.h>
  61 #include <net/if_arp.h>
  62 #include <net/if_dl.h>
  63 #include <net/if_media.h>
  64 #include <net/if_types.h>
  65 #include <net/if_vlan_var.h>
  66
  67 #include <netinet/in_systm.h>
  68 #include <netinet/in.h>
  69 #include <netinet/if_ether.h>
  70 #include <netinet/ip.h>
  71 #include <netinet/ip.h>
  72 #include <netinet/tcp.h>
  73 #include <netinet/udp.h>
  74
  75 #include <dev/pci/pcireg.h>
  76 #include <dev/pci/pcivar.h>
  77 #include <dev/pci/pci_private.h>
  78
  79 #ifdef CONFIG_DEFINED
  80 #include <cxgb_include.h>
  81 #else
  82 #include <dev/cxgb/cxgb_include.h>
  83 #endif
  84
  85 #ifdef PRIV_SUPPORTED
  86 #include <sys/priv.h>
  87 #endif
  88
  89 #ifdef IFNET_MULTIQUEUE
  90 #include <machine/intr_machdep.h>
  91 #endif
  92
  93 static int cxgb_setup_msix(adapter_t *, int);
  94 static void cxgb_teardown_msix(adapter_t *);
  95 static void cxgb_init(void *);
  96 static void cxgb_init_locked(struct port_info *);
  97 static void cxgb_stop_locked(struct port_info *);
  98 static void cxgb_set_rxmode(struct port_info *);
  99 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
 100 static int cxgb_media_change(struct ifnet *);
 101 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
 102 static int setup_sge_qsets(adapter_t *);
 103 static void cxgb_async_intr(void *);
 104 static void cxgb_ext_intr_handler(void *, int);
 105 static void cxgb_tick_handler(void *, int);
 106 static void cxgb_down_locked(struct adapter *sc);
 107 static void cxgb_tick(void *);
 108 static void setup_rss(adapter_t *sc);
 109
 110 /* Attachment glue for the PCI controller end of the device.  Each port of
 111  * the device is attached separately, as defined later.
 112  */
 113 static int cxgb_controller_probe(device_t);
 114 static int cxgb_controller_attach(device_t);
 115 static int cxgb_controller_detach(device_t);
 116 static void cxgb_free(struct adapter *);
 117 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
 118     unsigned int end);
 119 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
 120 static int cxgb_get_regs_len(void);
 121 static int offload_open(struct port_info *pi);
 122 static void touch_bars(device_t dev);
 123 static int offload_close(struct t3cdev *tdev);
 124 static void cxgb_link_start(struct port_info *p);
 125
 126 static device_method_t cxgb_controller_methods[] = {
 127         DEVMETHOD(device_probe,         cxgb_controller_probe),
 128         DEVMETHOD(device_attach,        cxgb_controller_attach),
 129         DEVMETHOD(device_detach,        cxgb_controller_detach),
 130
 131         /* bus interface */
 132         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 133         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 134
 135         { 0, 0 }
 136 };
 137
 138 static driver_t cxgb_controller_driver = {
 139         "cxgbc",
 140         cxgb_controller_methods,
 141         sizeof(struct adapter)
 142 };
 143
 144 static devclass_t       cxgb_controller_devclass;
 145 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
 146
 147 /*
 148  * Attachment glue for the ports.  Attachment is done directly to the
 149  * controller device.
 150  */
 151 static int cxgb_port_probe(device_t);
 152 static int cxgb_port_attach(device_t);
 153 static int cxgb_port_detach(device_t);
 154
 155 static device_method_t cxgb_port_methods[] = {
 156         DEVMETHOD(device_probe,         cxgb_port_probe),
 157         DEVMETHOD(device_attach,        cxgb_port_attach),
 158         DEVMETHOD(device_detach,        cxgb_port_detach),
 159         { 0, 0 }
 160 };
 161
 162 static driver_t cxgb_port_driver = {
 163         "cxgb",
 164         cxgb_port_methods,
 165         0
 166 };
 167
 168 static d_ioctl_t cxgb_extension_ioctl;
 169 static d_open_t cxgb_extension_open;
 170 static d_close_t cxgb_extension_close;
 171
 172 static struct cdevsw cxgb_cdevsw = {
 173        .d_version =    D_VERSION,
 174        .d_flags =      0,
 175        .d_open =       cxgb_extension_open,
 176        .d_close =      cxgb_extension_close,
 177        .d_ioctl =      cxgb_extension_ioctl,
 178        .d_name =       "cxgb",
 179 };
 180
 181 static devclass_t       cxgb_port_devclass;
 182 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
 183
 184 #define SGE_MSIX_COUNT (SGE_QSETS + 1)
 185
 186 /*
 187  * The driver uses the best interrupt scheme available on a platform in the
 188  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 189  * of these schemes the driver may consider as follows:
 190  *
 191  * msi = 2: choose from among all three options
 192  * msi = 1 : only consider MSI and pin interrupts
 193  * msi = 0: force pin interrupts
 194  */
 195 static int msi_allowed = 2;
 196
 197 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
 198 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
 199 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
 200     "MSI-X, MSI, INTx selector");
 201
 202 /*
 203  * The driver enables offload as a default.
 204  * To disable it, use ofld_disable = 1.
 205  */
 206 static int ofld_disable = 0;
 207 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
 208 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
 209     "disable ULP offload");
 210
 211 /*
 212  * The driver uses an auto-queue algorithm by default.
 213  * To disable it and force a single queue-set per port, use singleq = 1.
 214  */
 215 static int singleq = 0;
 216 TUNABLE_INT("hw.cxgb.singleq", &singleq);
 217 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
 218     "use a single queue-set per port");
 219
 220
 221 /*
 222  * The driver uses an auto-queue algorithm by default.
 223  * To disable it and force a single queue-set per port, use singleq = 1.
 224  */
 225 static int force_fw_update = 0;
 226 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
 227 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
 228     "update firmware even if up to date");
 229
 230 int cxgb_use_16k_clusters = 1;
 231 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
 232 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
 233     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
 234
 235 /*
 236  * Tune the size of the output queue.
 237  */
 238 int cxgb_snd_queue_len = IFQ_MAXLEN;
 239 TUNABLE_INT("hw.cxgb.snd_queue_len", &cxgb_snd_queue_len);
 240 SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_queue_len, CTLFLAG_RDTUN,
 241     &cxgb_snd_queue_len, 0, "send queue size ");
 242
 243
 244 enum {
 245         MAX_TXQ_ENTRIES      = 16384,
 246         MAX_CTRL_TXQ_ENTRIES = 1024,
 247         MAX_RSPQ_ENTRIES     = 16384,
 248         MAX_RX_BUFFERS       = 16384,
 249         MAX_RX_JUMBO_BUFFERS = 16384,
 250         MIN_TXQ_ENTRIES      = 4,
 251         MIN_CTRL_TXQ_ENTRIES = 4,
 252         MIN_RSPQ_ENTRIES     = 32,
 253         MIN_FL_ENTRIES       = 32,
 254         MIN_FL_JUMBO_ENTRIES = 32
 255 };
 256
 257 struct filter_info {
 258         u32 sip;
 259         u32 sip_mask;
 260         u32 dip;
 261         u16 sport;
 262         u16 dport;
 263         u32 vlan:12;
 264         u32 vlan_prio:3;
 265         u32 mac_hit:1;
 266         u32 mac_idx:4;
 267         u32 mac_vld:1;
 268         u32 pkt_type:2;
 269         u32 report_filter_id:1;
 270         u32 pass:1;
 271         u32 rss:1;
 272         u32 qset:3;
 273         u32 locked:1;
 274         u32 valid:1;
 275 };
 276
 277 enum { FILTER_NO_VLAN_PRI = 7 };
 278
 279 #define EEPROM_MAGIC 0x38E2F10C
 280
 281 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
 282
 283 /* Table for probing the cards.  The desc field isn't actually used */
 284 struct cxgb_ident {
 285         uint16_t        vendor;
 286         uint16_t        device;
 287         int             index;
 288         char            *desc;
 289 } cxgb_identifiers[] = {
 290         {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
 291         {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
 292         {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
 293         {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
 294         {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
 295         {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
 296         {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
 297         {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
 298         {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
 299         {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
 300         {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
 301         {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "N310E"},
 302         {0, 0, 0, NULL}
 303 };
 304
 305 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
 306
 307
 308 void
 309 cxgb_log_tcb(struct adapter *sc, unsigned int tid)
 310 {
 311         char buf[TCB_SIZE];
 312         uint64_t *tcb = (uint64_t *)buf;
 313         int i, error;
 314         struct mc7 *mem = &sc->cm;
 315
 316         error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
 317         if (error)
 318                 printf("cxgb_tcb_log failed\n");
 319
 320         CTR1(KTR_CXGB, "TCB tid=%u", tid);
 321         for (i = 0; i < TCB_SIZE / 32; i++) {
 322                 CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
 323                     i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
 324                     (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
 325                 tcb += 2;
 326                 CTR4(KTR_CXGB, "   %08x %08x %08x %08x",
 327                     (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
 328                     (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
 329                 tcb += 2;
 330         }
 331 }
 332
 333 static __inline char
 334 t3rev2char(struct adapter *adapter)
 335 {
 336         char rev = 'z';
 337
 338         switch(adapter->params.rev) {
 339         case T3_REV_A:
 340                 rev = 'a';
 341                 break;
 342         case T3_REV_B:
 343         case T3_REV_B2:
 344                 rev = 'b';
 345                 break;
 346         case T3_REV_C:
 347                 rev = 'c';
 348                 break;
 349         }
 350         return rev;
 351 }
 352
 353 static struct cxgb_ident *
 354 cxgb_get_ident(device_t dev)
 355 {
 356         struct cxgb_ident *id;
 357
 358         for (id = cxgb_identifiers; id->desc != NULL; id++) {
 359                 if ((id->vendor == pci_get_vendor(dev)) &&
 360                     (id->device == pci_get_device(dev))) {
 361                         return (id);
 362                 }
 363         }
 364         return (NULL);
 365 }
 366
 367 static const struct adapter_info *
 368 cxgb_get_adapter_info(device_t dev)
 369 {
 370         struct cxgb_ident *id;
 371         const struct adapter_info *ai;
 372
 373         id = cxgb_get_ident(dev);
 374         if (id == NULL)
 375                 return (NULL);
 376
 377         ai = t3_get_adapter_info(id->index);
 378
 379         return (ai);
 380 }
 381
 382 static int
 383 cxgb_controller_probe(device_t dev)
 384 {
 385         const struct adapter_info *ai;
 386         char *ports, buf[80];
 387         int nports;
 388         struct adapter *sc = device_get_softc(dev);
 389
 390         ai = cxgb_get_adapter_info(dev);
 391         if (ai == NULL)
 392                 return (ENXIO);
 393
 394         nports = ai->nports0 + ai->nports1;
 395         if (nports == 1)
 396                 ports = "port";
 397         else
 398                 ports = "ports";
 399
 400         snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s",
 401                  ai->desc, is_offload(sc) ? "R" : "",
 402                  sc->params.rev, nports, ports);
 403         device_set_desc_copy(dev, buf);
 404         return (BUS_PROBE_DEFAULT);
 405 }
 406
 407 #define FW_FNAME "cxgb_t3fw"
 408 #define TPEEPROM_NAME "t3b_tp_eeprom"
 409 #define TPSRAM_NAME "t3b_protocol_sram"
 410
 411 static int
 412 upgrade_fw(adapter_t *sc)
 413 {
 414 #ifdef FIRMWARE_LATEST
 415         const struct firmware *fw;
 416 #else
 417         struct firmware *fw;
 418 #endif
 419         int status;
 420
 421         if ((fw = firmware_get(FW_FNAME)) == NULL)  {
 422                 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
 423                 return (ENOENT);
 424         } else
 425                 device_printf(sc->dev, "updating firmware on card\n");
 426         status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
 427
 428         device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status);
 429
 430         firmware_put(fw, FIRMWARE_UNLOAD);
 431
 432         return (status);
 433 }
 434
 435 static int
 436 cxgb_controller_attach(device_t dev)
 437 {
 438         device_t child;
 439         const struct adapter_info *ai;
 440         struct adapter *sc;
 441         int i, error = 0;
 442         uint32_t vers;
 443         int port_qsets = 1;
 444 #ifdef MSI_SUPPORTED
 445         int msi_needed, reg;
 446 #endif
 447         int must_load = 0;
 448         char buf[80];
 449
 450         sc = device_get_softc(dev);
 451         sc->dev = dev;
 452         sc->msi_count = 0;
 453         ai = cxgb_get_adapter_info(dev);
 454
 455         /*
 456          * XXX not really related but a recent addition
 457          */
 458 #ifdef MSI_SUPPORTED
 459         /* find the PCIe link width and set max read request to 4KB*/
 460         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 461                 uint16_t lnk, pectl;
 462                 lnk = pci_read_config(dev, reg + 0x12, 2);
 463                 sc->link_width = (lnk >> 4) & 0x3f;
 464
 465                 pectl = pci_read_config(dev, reg + 0x8, 2);
 466                 pectl = (pectl & ~0x7000) | (5 << 12);
 467                 pci_write_config(dev, reg + 0x8, pectl, 2);
 468         }
 469
 470         if (sc->link_width != 0 && sc->link_width <= 4 &&
 471             (ai->nports0 + ai->nports1) <= 2) {
 472                 device_printf(sc->dev,
 473                     "PCIe x%d Link, expect reduced performance\n",
 474                     sc->link_width);
 475         }
 476 #endif
 477         touch_bars(dev);
 478         pci_enable_busmaster(dev);
 479         /*
 480          * Allocate the registers and make them available to the driver.
 481          * The registers that we care about for NIC mode are in BAR 0
 482          */
 483         sc->regs_rid = PCIR_BAR(0);
 484         if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 485             &sc->regs_rid, RF_ACTIVE)) == NULL) {
 486                 device_printf(dev, "Cannot allocate BAR region 0\n");
 487                 return (ENXIO);
 488         }
 489         sc->udbs_rid = PCIR_BAR(2);
 490         sc->udbs_res = NULL;
 491         if (is_offload(sc) &&
 492             ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 493                    &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
 494                 device_printf(dev, "Cannot allocate BAR region 1\n");
 495                 error = ENXIO;
 496                 goto out;
 497         }
 498
 499         snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
 500             device_get_unit(dev));
 501         ADAPTER_LOCK_INIT(sc, sc->lockbuf);
 502
 503         snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
 504             device_get_unit(dev));
 505         snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
 506             device_get_unit(dev));
 507         snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
 508             device_get_unit(dev));
 509
 510         MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
 511         MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
 512         MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
 513
 514         sc->bt = rman_get_bustag(sc->regs_res);
 515         sc->bh = rman_get_bushandle(sc->regs_res);
 516         sc->mmio_len = rman_get_size(sc->regs_res);
 517
 518         if (t3_prep_adapter(sc, ai, 1) < 0) {
 519                 printf("prep adapter failed\n");
 520                 error = ENODEV;
 521                 goto out;
 522         }
 523         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
 524          * enough messages for the queue sets.  If that fails, try falling
 525          * back to MSI.  If that fails, then try falling back to the legacy
 526          * interrupt pin model.
 527          */
 528 #ifdef MSI_SUPPORTED
 529
 530         sc->msix_regs_rid = 0x20;
 531         if ((msi_allowed >= 2) &&
 532             (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 533             &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
 534
 535                 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
 536
 537                 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
 538                     (sc->msi_count != msi_needed)) {
 539                         device_printf(dev, "msix allocation failed - msi_count = %d"
 540                             " msi_needed=%d will try msi err=%d\n", sc->msi_count,
 541                             msi_needed, error);
 542                         sc->msi_count = 0;
 543                         pci_release_msi(dev);
 544                         bus_release_resource(dev, SYS_RES_MEMORY,
 545                             sc->msix_regs_rid, sc->msix_regs_res);
 546                         sc->msix_regs_res = NULL;
 547                 } else {
 548                         sc->flags |= USING_MSIX;
 549                         sc->cxgb_intr = t3_intr_msix;
 550                 }
 551         }
 552
 553         if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
 554                 sc->msi_count = 1;
 555                 if (pci_alloc_msi(dev, &sc->msi_count)) {
 556                         device_printf(dev, "alloc msi failed - will try INTx\n");
 557                         sc->msi_count = 0;
 558                         pci_release_msi(dev);
 559                 } else {
 560                         sc->flags |= USING_MSI;
 561                         sc->irq_rid = 1;
 562                         sc->cxgb_intr = t3_intr_msi;
 563                 }
 564         }
 565 #endif
 566         if (sc->msi_count == 0) {
 567                 device_printf(dev, "using line interrupts\n");
 568                 sc->irq_rid = 0;
 569                 sc->cxgb_intr = t3b_intr;
 570         }
 571
 572         if ((sc->flags & USING_MSIX) && !singleq)
 573                 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
 574
 575         /* Create a private taskqueue thread for handling driver events */
 576 #ifdef TASKQUEUE_CURRENT
 577         sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
 578             taskqueue_thread_enqueue, &sc->tq);
 579 #else
 580         sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
 581             taskqueue_thread_enqueue, &sc->tq);
 582 #endif
 583         if (sc->tq == NULL) {
 584                 device_printf(dev, "failed to allocate controller task queue\n");
 585                 goto out;
 586         }
 587
 588         taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
 589             device_get_nameunit(dev));
 590         TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
 591         TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
 592
 593
 594         /* Create a periodic callout for checking adapter status */
 595         callout_init(&sc->cxgb_tick_ch, TRUE);
 596
 597         if ((t3_check_fw_version(sc, &must_load) != 0 && must_load) || force_fw_update) {
 598                 /*
 599                  * Warn user that a firmware update will be attempted in init.
 600                  */
 601                 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
 602                     FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 603                 sc->flags &= ~FW_UPTODATE;
 604         } else {
 605                 sc->flags |= FW_UPTODATE;
 606         }
 607
 608         if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) {
 609                 /*
 610                  * Warn user that a firmware update will be attempted in init.
 611                  */
 612                 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
 613                     t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 614                 sc->flags &= ~TPS_UPTODATE;
 615         } else {
 616                 sc->flags |= TPS_UPTODATE;
 617         }
 618
 619         /*
 620          * Create a child device for each MAC.  The ethernet attachment
 621          * will be done in these children.
 622          */
 623         for (i = 0; i < (sc)->params.nports; i++) {
 624                 struct port_info *pi;
 625
 626                 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
 627                         device_printf(dev, "failed to add child port\n");
 628                         error = EINVAL;
 629                         goto out;
 630                 }
 631                 pi = &sc->port[i];
 632                 pi->adapter = sc;
 633                 pi->nqsets = port_qsets;
 634                 pi->first_qset = i*port_qsets;
 635                 pi->port_id = i;
 636                 pi->tx_chan = i >= ai->nports0;
 637                 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
 638                 sc->rxpkt_map[pi->txpkt_intf] = i;
 639                 sc->port[i].tx_chan = i >= ai->nports0;
 640                 sc->portdev[i] = child;
 641                 device_set_softc(child, pi);
 642         }
 643         if ((error = bus_generic_attach(dev)) != 0)
 644                 goto out;
 645
 646         /* initialize sge private state */
 647         t3_sge_init_adapter(sc);
 648
 649         t3_led_ready(sc);
 650
 651         cxgb_offload_init();
 652         if (is_offload(sc)) {
 653                 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
 654                 cxgb_adapter_ofld(sc);
 655         }
 656         error = t3_get_fw_version(sc, &vers);
 657         if (error)
 658                 goto out;
 659
 660         snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
 661             G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
 662             G_FW_VERSION_MICRO(vers));
 663
 664         snprintf(buf, sizeof(buf), "%s\t E/C: %s S/N: %s",
 665                  ai->desc,
 666                  sc->params.vpd.ec, sc->params.vpd.sn);
 667         device_set_desc_copy(dev, buf);
 668
 669         device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
 670         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
 671         t3_add_attach_sysctls(sc);
 672 out:
 673         if (error)
 674                 cxgb_free(sc);
 675
 676         return (error);
 677 }
 678
 679 static int
 680 cxgb_controller_detach(device_t dev)
 681 {
 682         struct adapter *sc;
 683
 684         sc = device_get_softc(dev);
 685
 686         cxgb_free(sc);
 687
 688         return (0);
 689 }
 690
 691 static void
 692 cxgb_free(struct adapter *sc)
 693 {
 694         int i;
 695
 696         ADAPTER_LOCK(sc);
 697         sc->flags |= CXGB_SHUTDOWN;
 698         ADAPTER_UNLOCK(sc);
 699         cxgb_pcpu_shutdown_threads(sc);
 700         ADAPTER_LOCK(sc);
 701
 702 /*
 703  * drops the lock
 704  */
 705         cxgb_down_locked(sc);
 706
 707 #ifdef MSI_SUPPORTED
 708         if (sc->flags & (USING_MSI | USING_MSIX)) {
 709                 device_printf(sc->dev, "releasing msi message(s)\n");
 710                 pci_release_msi(sc->dev);
 711         } else {
 712                 device_printf(sc->dev, "no msi message to release\n");
 713         }
 714 #endif
 715         if (sc->msix_regs_res != NULL) {
 716                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
 717                     sc->msix_regs_res);
 718         }
 719
 720         t3_sge_deinit_sw(sc);
 721         /*
 722          * Wait for last callout
 723          */
 724
 725         DELAY(hz*100);
 726
 727         for (i = 0; i < (sc)->params.nports; ++i) {
 728                 if (sc->portdev[i] != NULL)
 729                         device_delete_child(sc->dev, sc->portdev[i]);
 730         }
 731
 732         bus_generic_detach(sc->dev);
 733         if (sc->tq != NULL) {
 734                 taskqueue_free(sc->tq);
 735                 sc->tq = NULL;
 736         }
 737
 738         if (is_offload(sc)) {
 739                 cxgb_adapter_unofld(sc);
 740                 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
 741                         offload_close(&sc->tdev);
 742                 else
 743                         printf("cxgb_free: DEVMAP_BIT not set\n");
 744         } else
 745                 printf("not offloading set\n");
 746 #ifdef notyet
 747         if (sc->flags & CXGB_OFLD_INIT)
 748                 cxgb_offload_deactivate(sc);
 749 #endif
 750         free(sc->filters, M_DEVBUF);
 751         t3_sge_free(sc);
 752
 753         cxgb_offload_exit();
 754
 755         if (sc->udbs_res != NULL)
 756                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
 757                     sc->udbs_res);
 758
 759         if (sc->regs_res != NULL)
 760                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
 761                     sc->regs_res);
 762
 763         MTX_DESTROY(&sc->mdio_lock);
 764         MTX_DESTROY(&sc->sge.reg_lock);
 765         MTX_DESTROY(&sc->elmer_lock);
 766         ADAPTER_LOCK_DEINIT(sc);
 767 }
 768
 769 /**
 770  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 771  *      @sc: the controller softc
 772  *
 773  *      Determines how many sets of SGE queues to use and initializes them.
 774  *      We support multiple queue sets per port if we have MSI-X, otherwise
 775  *      just one queue set per port.
 776  */
 777 static int
 778 setup_sge_qsets(adapter_t *sc)
 779 {
 780         int i, j, err, irq_idx = 0, qset_idx = 0;
 781         u_int ntxq = SGE_TXQ_PER_SET;
 782
 783         if ((err = t3_sge_alloc(sc)) != 0) {
 784                 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
 785                 return (err);
 786         }
 787
 788         if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
 789                 irq_idx = -1;
 790
 791         for (i = 0; i < (sc)->params.nports; i++) {
 792                 struct port_info *pi = &sc->port[i];
 793
 794                 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
 795                         err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
 796                             (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
 797                             &sc->params.sge.qset[qset_idx], ntxq, pi);
 798                         if (err) {
 799                                 t3_free_sge_resources(sc);
 800                                 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n",
 801                                     err);
 802                                 return (err);
 803                         }
 804                 }
 805         }
 806
 807         return (0);
 808 }
 809
 810 static void
 811 cxgb_teardown_msix(adapter_t *sc)
 812 {
 813         int i, nqsets;
 814
 815         for (nqsets = i = 0; i < (sc)->params.nports; i++)
 816                 nqsets += sc->port[i].nqsets;
 817
 818         for (i = 0; i < nqsets; i++) {
 819                 if (sc->msix_intr_tag[i] != NULL) {
 820                         bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
 821                             sc->msix_intr_tag[i]);
 822                         sc->msix_intr_tag[i] = NULL;
 823                 }
 824                 if (sc->msix_irq_res[i] != NULL) {
 825                         bus_release_resource(sc->dev, SYS_RES_IRQ,
 826                             sc->msix_irq_rid[i], sc->msix_irq_res[i]);
 827                         sc->msix_irq_res[i] = NULL;
 828                 }
 829         }
 830 }
 831
 832 static int
 833 cxgb_setup_msix(adapter_t *sc, int msix_count)
 834 {
 835         int i, j, k, nqsets, rid;
 836
 837         /* The first message indicates link changes and error conditions */
 838         sc->irq_rid = 1;
 839         if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
 840            &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
 841                 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
 842                 return (EINVAL);
 843         }
 844
 845         if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
 846 #ifdef INTR_FILTERS
 847                 NULL,
 848 #endif
 849                 cxgb_async_intr, sc, &sc->intr_tag)) {
 850                 device_printf(sc->dev, "Cannot set up interrupt\n");
 851                 return (EINVAL);
 852         }
 853         for (i = k = 0; i < (sc)->params.nports; i++) {
 854                 nqsets = sc->port[i].nqsets;
 855                 for (j = 0; j < nqsets; j++, k++) {
 856                         struct sge_qset *qs = &sc->sge.qs[k];
 857
 858                         rid = k + 2;
 859                         if (cxgb_debug)
 860                                 printf("rid=%d ", rid);
 861                         if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
 862                             sc->dev, SYS_RES_IRQ, &rid,
 863                             RF_SHAREABLE | RF_ACTIVE)) == NULL) {
 864                                 device_printf(sc->dev, "Cannot allocate "
 865                                     "interrupt for message %d\n", rid);
 866                                 return (EINVAL);
 867                         }
 868                         sc->msix_irq_rid[k] = rid;
 869                         if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
 870                                 INTR_MPSAFE|INTR_TYPE_NET,
 871 #ifdef INTR_FILTERS
 872                                 NULL,
 873 #endif
 874                                 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
 875                                 device_printf(sc->dev, "Cannot set up "
 876                                     "interrupt for message %d\n", rid);
 877                                 return (EINVAL);
 878                         }
 879 #ifdef IFNET_MULTIQUEUE
 880                         if (singleq == 0) {
 881                                 int vector = rman_get_start(sc->msix_irq_res[k]);
 882                                 if (bootverbose)
 883                                         device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus);
 884                                 intr_bind(vector, k % mp_ncpus);
 885                         }
 886 #endif
 887                 }
 888         }
 889
 890         return (0);
 891 }
 892
 893 static int
 894 cxgb_port_probe(device_t dev)
 895 {
 896         struct port_info *p;
 897         char buf[80];
 898         const char *desc;
 899
 900         p = device_get_softc(dev);
 901         desc = p->phy.desc;
 902         snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
 903         device_set_desc_copy(dev, buf);
 904         return (0);
 905 }
 906
 907
 908 static int
 909 cxgb_makedev(struct port_info *pi)
 910 {
 911
 912         pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
 913             UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
 914
 915         if (pi->port_cdev == NULL)
 916                 return (ENOMEM);
 917
 918         pi->port_cdev->si_drv1 = (void *)pi;
 919
 920         return (0);
 921 }
 922
 923
 924 #ifdef TSO_SUPPORTED
 925 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO)
 926 /* Don't enable TSO6 yet */
 927 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO)
 928 #else
 929 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
 930 /* Don't enable TSO6 yet */
 931 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM |  IFCAP_JUMBO_MTU)
 932 #define IFCAP_TSO4 0x0
 933 #define IFCAP_TSO6 0x0
 934 #define CSUM_TSO   0x0
 935 #endif
 936
 937
 938 static int
 939 cxgb_port_attach(device_t dev)
 940 {
 941         struct port_info *p;
 942         struct ifnet *ifp;
 943         int err, media_flags;
 944         struct adapter *sc;
 945
 946
 947         p = device_get_softc(dev);
 948         sc = p->adapter;
 949         snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
 950             device_get_unit(device_get_parent(dev)), p->port_id);
 951         PORT_LOCK_INIT(p, p->lockbuf);
 952
 953         /* Allocate an ifnet object and set it up */
 954         ifp = p->ifp = if_alloc(IFT_ETHER);
 955         if (ifp == NULL) {
 956                 device_printf(dev, "Cannot allocate ifnet\n");
 957                 return (ENOMEM);
 958         }
 959
 960         /*
 961          * Note that there is currently no watchdog timer.
 962          */
 963         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 964         ifp->if_init = cxgb_init;
 965         ifp->if_softc = p;
 966         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 967         ifp->if_ioctl = cxgb_ioctl;
 968         ifp->if_start = cxgb_start;
 969
 970 #if 0
 971 #ifdef IFNET_MULTIQUEUE
 972         ifp->if_flags |= IFF_MULTIQ;
 973         ifp->if_mq_start = cxgb_pcpu_start;
 974 #endif
 975 #endif
 976         ifp->if_timer = 0;      /* Disable ifnet watchdog */
 977         ifp->if_watchdog = NULL;
 978
 979         ifp->if_snd.ifq_drv_maxlen = cxgb_snd_queue_len;
 980         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
 981         IFQ_SET_READY(&ifp->if_snd);
 982
 983         ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
 984         ifp->if_capabilities |= CXGB_CAP;
 985         ifp->if_capenable |= CXGB_CAP_ENABLE;
 986         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
 987         /*
 988          * disable TSO on 4-port - it isn't supported by the firmware yet
 989          */
 990         if (p->adapter->params.nports > 2) {
 991                 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6);
 992                 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6);
 993                 ifp->if_hwassist &= ~CSUM_TSO;
 994         }
 995
 996         ether_ifattach(ifp, p->hw_addr);
 997         /*
 998          * Only default to jumbo frames on 10GigE
 999          */
1000         if (p->adapter->params.nports <= 2)
1001                 ifp->if_mtu = ETHERMTU_JUMBO;
1002         if ((err = cxgb_makedev(p)) != 0) {
1003                 printf("makedev failed %d\n", err);
1004                 return (err);
1005         }
1006         ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1007             cxgb_media_status);
1008
1009         if (!strcmp(p->phy.desc,        "10GBASE-CX4")) {
1010                 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
1011         } else if (!strcmp(p->phy.desc, "10GBASE-SR")) {
1012                 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
1013         } else if (!strcmp(p->phy.desc, "10GBASE-R")) {
1014                 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
1015         } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) {
1016                 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
1017                 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
1018                             0, NULL);
1019                 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
1020                             0, NULL);
1021                 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1022                             0, NULL);
1023                 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1024                             0, NULL);
1025                 media_flags = 0;
1026         } else if (!strcmp(p->phy.desc, "1000BASE-X")) {
1027                 /*
1028                  * XXX: This is not very accurate.  Fix when common code
1029                  * returns more specific value - eg 1000BASE-SX, LX, etc.
1030                  */
1031                 media_flags = IFM_ETHER | IFM_1000_SX | IFM_FDX;
1032         } else {
1033                 printf("unsupported media type %s\n", p->phy.desc);
1034                 return (ENXIO);
1035         }
1036         if (media_flags) {
1037                 ifmedia_add(&p->media, media_flags, 0, NULL);
1038                 ifmedia_set(&p->media, media_flags);
1039         } else {
1040                 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1041                 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
1042         }
1043
1044         /* Get the latest mac address, User can use a LAA */
1045         bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN);
1046         t3_sge_init_port(p);
1047 #if defined(LINK_ATTACH)
1048         cxgb_link_start(p);
1049         t3_link_changed(sc, p->port_id);
1050 #endif
1051         return (0);
1052 }
1053
1054 static int
1055 cxgb_port_detach(device_t dev)
1056 {
1057         struct port_info *p;
1058
1059         p = device_get_softc(dev);
1060
1061         PORT_LOCK(p);
1062         if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
1063                 cxgb_stop_locked(p);
1064         PORT_UNLOCK(p);
1065
1066         ether_ifdetach(p->ifp);
1067         printf("waiting for callout to stop ...");
1068         DELAY(1000000);
1069         printf("done\n");
1070         /*
1071          * the lock may be acquired in ifdetach
1072          */
1073         PORT_LOCK_DEINIT(p);
1074         if_free(p->ifp);
1075
1076         if (p->port_cdev != NULL)
1077                 destroy_dev(p->port_cdev);
1078
1079         return (0);
1080 }
1081
1082 void
1083 t3_fatal_err(struct adapter *sc)
1084 {
1085         u_int fw_status[4];
1086         int i = 0;
1087
1088         /*
1089          * We don't know which tcb caused the error so we just hope it was one of the first ten :-/
1090          */
1091         for (i = 0; i < 10; i++)
1092                 cxgb_log_tcb(sc, i);
1093
1094         if (sc->flags & FULL_INIT_DONE) {
1095                 t3_sge_stop(sc);
1096                 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1097                 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1098                 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1099                 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1100                 t3_intr_disable(sc);
1101         }
1102         device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1103         if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status)) {
1104
1105                 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1106                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1107
1108                 CTR4(KTR_CXGB, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1109                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1110         }
1111 }
1112
1113 int
1114 t3_os_find_pci_capability(adapter_t *sc, int cap)
1115 {
1116         device_t dev;
1117         struct pci_devinfo *dinfo;
1118         pcicfgregs *cfg;
1119         uint32_t status;
1120         uint8_t ptr;
1121
1122         dev = sc->dev;
1123         dinfo = device_get_ivars(dev);
1124         cfg = &dinfo->cfg;
1125
1126         status = pci_read_config(dev, PCIR_STATUS, 2);
1127         if (!(status & PCIM_STATUS_CAPPRESENT))
1128                 return (0);
1129
1130         switch (cfg->hdrtype & PCIM_HDRTYPE) {
1131         case 0:
1132         case 1:
1133                 ptr = PCIR_CAP_PTR;
1134                 break;
1135         case 2:
1136                 ptr = PCIR_CAP_PTR_2;
1137                 break;
1138         default:
1139                 return (0);
1140                 break;
1141         }
1142         ptr = pci_read_config(dev, ptr, 1);
1143
1144         while (ptr != 0) {
1145                 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1146                         return (ptr);
1147                 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1148         }
1149
1150         return (0);
1151 }
1152
1153 int
1154 t3_os_pci_save_state(struct adapter *sc)
1155 {
1156         device_t dev;
1157         struct pci_devinfo *dinfo;
1158
1159         dev = sc->dev;
1160         dinfo = device_get_ivars(dev);
1161
1162         pci_cfg_save(dev, dinfo, 0);
1163         return (0);
1164 }
1165
1166 int
1167 t3_os_pci_restore_state(struct adapter *sc)
1168 {
1169         device_t dev;
1170         struct pci_devinfo *dinfo;
1171
1172         dev = sc->dev;
1173         dinfo = device_get_ivars(dev);
1174
1175         pci_cfg_restore(dev, dinfo);
1176         return (0);
1177 }
1178
1179 /**
1180  *      t3_os_link_changed - handle link status changes
1181  *      @adapter: the adapter associated with the link change
1182  *      @port_id: the port index whose limk status has changed
1183  *      @link_status: the new status of the link
1184  *      @speed: the new speed setting
1185  *      @duplex: the new duplex setting
1186  *      @fc: the new flow-control setting
1187  *
1188  *      This is the OS-dependent handler for link status changes.  The OS
1189  *      neutral handler takes care of most of the processing for these events,
1190  *      then calls this handler for any OS-specific processing.
1191  */
1192 void
1193 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1194      int duplex, int fc)
1195 {
1196         struct port_info *pi = &adapter->port[port_id];
1197         struct cmac *mac = &adapter->port[port_id].mac;
1198
1199         if (link_status) {
1200                 DELAY(10);
1201                 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1202                         /* Clear errors created by MAC enable */
1203                         t3_set_reg_field(adapter,
1204                                          A_XGM_STAT_CTRL + pi->mac.offset,
1205                                          F_CLRSTATS, 1);
1206                 if_link_state_change(pi->ifp, LINK_STATE_UP);
1207
1208         } else {
1209                 pi->phy.ops->power_down(&pi->phy, 1);
1210                 t3_mac_disable(mac, MAC_DIRECTION_RX);
1211                 t3_link_start(&pi->phy, mac, &pi->link_config);
1212                 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1213                 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
1214         }
1215 }
1216
1217 /**
1218  *      t3_os_phymod_changed - handle PHY module changes
1219  *      @phy: the PHY reporting the module change
1220  *      @mod_type: new module type
1221  *
1222  *      This is the OS-dependent handler for PHY module changes.  It is
1223  *      invoked when a PHY module is removed or inserted for any OS-specific
1224  *      processing.
1225  */
1226 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1227 {
1228         static const char *mod_str[] = {
1229                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
1230         };
1231
1232         struct port_info *pi = &adap->port[port_id];
1233
1234         if (pi->phy.modtype == phy_modtype_none)
1235                 device_printf(adap->dev, "PHY module unplugged\n");
1236         else {
1237                 KASSERT(pi->phy.modtype < ARRAY_SIZE(mod_str),
1238                     ("invalid PHY module type %d", pi->phy.modtype));
1239                 device_printf(adap->dev, "%s PHY module inserted\n",
1240                     mod_str[pi->phy.modtype]);
1241         }
1242 }
1243
1244 /*
1245  * Interrupt-context handler for external (PHY) interrupts.
1246  */
1247 void
1248 t3_os_ext_intr_handler(adapter_t *sc)
1249 {
1250         if (cxgb_debug)
1251                 printf("t3_os_ext_intr_handler\n");
1252         /*
1253          * Schedule a task to handle external interrupts as they may be slow
1254          * and we use a mutex to protect MDIO registers.  We disable PHY
1255          * interrupts in the meantime and let the task reenable them when
1256          * it's done.
1257          */
1258         ADAPTER_LOCK(sc);
1259         if (sc->slow_intr_mask) {
1260                 sc->slow_intr_mask &= ~F_T3DBG;
1261                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1262                 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1263         }
1264         ADAPTER_UNLOCK(sc);
1265 }
1266
1267 void
1268 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1269 {
1270
1271         /*
1272          * The ifnet might not be allocated before this gets called,
1273          * as this is called early on in attach by t3_prep_adapter
1274          * save the address off in the port structure
1275          */
1276         if (cxgb_debug)
1277                 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1278         bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1279 }
1280
1281 /**
1282  *      link_start - enable a port
1283  *      @p: the port to enable
1284  *
1285  *      Performs the MAC and PHY actions needed to enable a port.
1286  */
1287 static void
1288 cxgb_link_start(struct port_info *p)
1289 {
1290         struct ifnet *ifp;
1291         struct t3_rx_mode rm;
1292         struct cmac *mac = &p->mac;
1293         int mtu, hwtagging;
1294
1295         ifp = p->ifp;
1296
1297         bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1298
1299         mtu = ifp->if_mtu;
1300         if (ifp->if_capenable & IFCAP_VLAN_MTU)
1301                 mtu += ETHER_VLAN_ENCAP_LEN;
1302
1303         hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1304
1305         t3_init_rx_mode(&rm, p);
1306         if (!mac->multiport)
1307                 t3_mac_reset(mac);
1308         t3_mac_set_mtu(mac, mtu);
1309         t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1310         t3_mac_set_address(mac, 0, p->hw_addr);
1311         t3_mac_set_rx_mode(mac, &rm);
1312         t3_link_start(&p->phy, mac, &p->link_config);
1313         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1314 }
1315
1316
1317 static int
1318 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1319                               unsigned long n)
1320 {
1321         int attempts = 5;
1322
1323         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1324                 if (!--attempts)
1325                         return (ETIMEDOUT);
1326                 t3_os_sleep(10);
1327         }
1328         return 0;
1329 }
1330
1331 static int
1332 init_tp_parity(struct adapter *adap)
1333 {
1334         int i;
1335         struct mbuf *m;
1336         struct cpl_set_tcb_field *greq;
1337         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1338
1339         t3_tp_set_offload_mode(adap, 1);
1340
1341         for (i = 0; i < 16; i++) {
1342                 struct cpl_smt_write_req *req;
1343
1344                 m = m_gethdr(M_WAITOK, MT_DATA);
1345                 req = mtod(m, struct cpl_smt_write_req *);
1346                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1347                 memset(req, 0, sizeof(*req));
1348                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1349                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1350                 req->iff = i;
1351                 t3_mgmt_tx(adap, m);
1352         }
1353
1354         for (i = 0; i < 2048; i++) {
1355                 struct cpl_l2t_write_req *req;
1356
1357                 m = m_gethdr(M_WAITOK, MT_DATA);
1358                 req = mtod(m, struct cpl_l2t_write_req *);
1359                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1360                 memset(req, 0, sizeof(*req));
1361                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1362                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1363                 req->params = htonl(V_L2T_W_IDX(i));
1364                 t3_mgmt_tx(adap, m);
1365         }
1366
1367         for (i = 0; i < 2048; i++) {
1368                 struct cpl_rte_write_req *req;
1369
1370                 m = m_gethdr(M_WAITOK, MT_DATA);
1371                 req = mtod(m, struct cpl_rte_write_req *);
1372                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1373                 memset(req, 0, sizeof(*req));
1374                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1375                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1376                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1377                 t3_mgmt_tx(adap, m);
1378         }
1379
1380         m = m_gethdr(M_WAITOK, MT_DATA);
1381         greq = mtod(m, struct cpl_set_tcb_field *);
1382         m->m_len = m->m_pkthdr.len = sizeof(*greq);
1383         memset(greq, 0, sizeof(*greq));
1384         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1385         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1386         greq->mask = htobe64(1);
1387         t3_mgmt_tx(adap, m);
1388
1389         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1390         t3_tp_set_offload_mode(adap, 0);
1391         return (i);
1392 }
1393
1394 /**
1395  *      setup_rss - configure Receive Side Steering (per-queue connection demux)
1396  *      @adap: the adapter
1397  *
1398  *      Sets up RSS to distribute packets to multiple receive queues.  We
1399  *      configure the RSS CPU lookup table to distribute to the number of HW
1400  *      receive queues, and the response queue lookup table to narrow that
1401  *      down to the response queues actually configured for each port.
1402  *      We always configure the RSS mapping for two ports since the mapping
1403  *      table has plenty of entries.
1404  */
1405 static void
1406 setup_rss(adapter_t *adap)
1407 {
1408         int i;
1409         u_int nq[2];
1410         uint8_t cpus[SGE_QSETS + 1];
1411         uint16_t rspq_map[RSS_TABLE_SIZE];
1412
1413         for (i = 0; i < SGE_QSETS; ++i)
1414                 cpus[i] = i;
1415         cpus[SGE_QSETS] = 0xff;
1416
1417         nq[0] = nq[1] = 0;
1418         for_each_port(adap, i) {
1419                 const struct port_info *pi = adap2pinfo(adap, i);
1420
1421                 nq[pi->tx_chan] += pi->nqsets;
1422         }
1423         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1424                 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1425                 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1426         }
1427         /* Calculate the reverse RSS map table */
1428         for (i = 0; i < RSS_TABLE_SIZE; ++i)
1429                 if (adap->rrss_map[rspq_map[i]] == 0xff)
1430                         adap->rrss_map[rspq_map[i]] = i;
1431
1432         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1433                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1434                       F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1435                       cpus, rspq_map);
1436
1437 }
1438
1439 /*
1440  * Sends an mbuf to an offload queue driver
1441  * after dealing with any active network taps.
1442  */
1443 static inline int
1444 offload_tx(struct t3cdev *tdev, struct mbuf *m)
1445 {
1446         int ret;
1447
1448         ret = t3_offload_tx(tdev, m);
1449         return (ret);
1450 }
1451
1452 static int
1453 write_smt_entry(struct adapter *adapter, int idx)
1454 {
1455         struct port_info *pi = &adapter->port[idx];
1456         struct cpl_smt_write_req *req;
1457         struct mbuf *m;
1458
1459         if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1460                 return (ENOMEM);
1461
1462         req = mtod(m, struct cpl_smt_write_req *);
1463         m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
1464
1465         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1466         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1467         req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
1468         req->iff = idx;
1469         memset(req->src_mac1, 0, sizeof(req->src_mac1));
1470         memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1471
1472         m_set_priority(m, 1);
1473
1474         offload_tx(&adapter->tdev, m);
1475
1476         return (0);
1477 }
1478
1479 static int
1480 init_smt(struct adapter *adapter)
1481 {
1482         int i;
1483
1484         for_each_port(adapter, i)
1485                 write_smt_entry(adapter, i);
1486         return 0;
1487 }
1488
1489 static void
1490 init_port_mtus(adapter_t *adapter)
1491 {
1492         unsigned int mtus = adapter->port[0].ifp->if_mtu;
1493
1494         if (adapter->port[1].ifp)
1495                 mtus |= adapter->port[1].ifp->if_mtu << 16;
1496         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1497 }
1498
1499 static void
1500 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1501                               int hi, int port)
1502 {
1503         struct mbuf *m;
1504         struct mngt_pktsched_wr *req;
1505
1506         m = m_gethdr(M_DONTWAIT, MT_DATA);
1507         if (m) {
1508                 req = mtod(m, struct mngt_pktsched_wr *);
1509                 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1510                 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1511                 req->sched = sched;
1512                 req->idx = qidx;
1513                 req->min = lo;
1514                 req->max = hi;
1515                 req->binding = port;
1516                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1517                 t3_mgmt_tx(adap, m);
1518         }
1519 }
1520
1521 static void
1522 bind_qsets(adapter_t *sc)
1523 {
1524         int i, j;
1525
1526         cxgb_pcpu_startup_threads(sc);
1527         for (i = 0; i < (sc)->params.nports; ++i) {
1528                 const struct port_info *pi = adap2pinfo(sc, i);
1529
1530                 for (j = 0; j < pi->nqsets; ++j) {
1531                         send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1532                                           -1, pi->tx_chan);
1533
1534                 }
1535         }
1536 }
1537
1538 static void
1539 update_tpeeprom(struct adapter *adap)
1540 {
1541 #ifdef FIRMWARE_LATEST
1542         const struct firmware *tpeeprom;
1543 #else
1544         struct firmware *tpeeprom;
1545 #endif
1546
1547         uint32_t version;
1548         unsigned int major, minor;
1549         int ret, len;
1550         char rev;
1551
1552         t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1553
1554         major = G_TP_VERSION_MAJOR(version);
1555         minor = G_TP_VERSION_MINOR(version);
1556         if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
1557                 return;
1558
1559         rev = t3rev2char(adap);
1560
1561         tpeeprom = firmware_get(TPEEPROM_NAME);
1562         if (tpeeprom == NULL) {
1563                 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n",
1564                     TPEEPROM_NAME);
1565                 return;
1566         }
1567
1568         len = tpeeprom->datasize - 4;
1569
1570         ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1571         if (ret)
1572                 goto release_tpeeprom;
1573
1574         if (len != TP_SRAM_LEN) {
1575                 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", TPEEPROM_NAME, len, TP_SRAM_LEN);
1576                 return;
1577         }
1578
1579         ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1580             TP_SRAM_OFFSET);
1581
1582         if (!ret) {
1583                 device_printf(adap->dev,
1584                         "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1585                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1586         } else
1587                 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n");
1588
1589 release_tpeeprom:
1590         firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1591
1592         return;
1593 }
1594
1595 static int
1596 update_tpsram(struct adapter *adap)
1597 {
1598 #ifdef FIRMWARE_LATEST
1599         const struct firmware *tpsram;
1600 #else
1601         struct firmware *tpsram;
1602 #endif
1603         int ret;
1604         char rev;
1605
1606         rev = t3rev2char(adap);
1607         if (!rev)
1608                 return 0;
1609
1610         update_tpeeprom(adap);
1611
1612         tpsram = firmware_get(TPSRAM_NAME);
1613         if (tpsram == NULL){
1614                 device_printf(adap->dev, "could not load TP SRAM\n");
1615                 return (EINVAL);
1616         } else
1617                 device_printf(adap->dev, "updating TP SRAM\n");
1618
1619         ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1620         if (ret)
1621                 goto release_tpsram;
1622
1623         ret = t3_set_proto_sram(adap, tpsram->data);
1624         if (ret)
1625                 device_printf(adap->dev, "loading protocol SRAM failed\n");
1626
1627 release_tpsram:
1628         firmware_put(tpsram, FIRMWARE_UNLOAD);
1629
1630         return ret;
1631 }
1632
1633 /**
1634  *      cxgb_up - enable the adapter
1635  *      @adap: adapter being enabled
1636  *
1637  *      Called when the first port is enabled, this function performs the
1638  *      actions necessary to make an adapter operational, such as completing
1639  *      the initialization of HW modules, and enabling interrupts.
1640  *
1641  */
1642 static int
1643 cxgb_up(struct adapter *sc)
1644 {
1645         int err = 0;
1646
1647         if ((sc->flags & FULL_INIT_DONE) == 0) {
1648
1649                 if ((sc->flags & FW_UPTODATE) == 0)
1650                         if ((err = upgrade_fw(sc)))
1651                                 goto out;
1652                 if ((sc->flags & TPS_UPTODATE) == 0)
1653                         if ((err = update_tpsram(sc)))
1654                                 goto out;
1655                 err = t3_init_hw(sc, 0);
1656                 if (err)
1657                         goto out;
1658
1659                 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1660                 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1661
1662                 err = setup_sge_qsets(sc);
1663                 if (err)
1664                         goto out;
1665
1666                 setup_rss(sc);
1667                 t3_add_configured_sysctls(sc);
1668                 sc->flags |= FULL_INIT_DONE;
1669         }
1670
1671         t3_intr_clear(sc);
1672
1673         /* If it's MSI or INTx, allocate a single interrupt for everything */
1674         if ((sc->flags & USING_MSIX) == 0) {
1675                 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1676                    &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1677                         device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n",
1678                             sc->irq_rid);
1679                         err = EINVAL;
1680                         goto out;
1681                 }
1682                 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1683
1684                 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1685 #ifdef INTR_FILTERS
1686                         NULL,
1687 #endif
1688                         sc->cxgb_intr, sc, &sc->intr_tag)) {
1689                         device_printf(sc->dev, "Cannot set up interrupt\n");
1690                         err = EINVAL;
1691                         goto irq_err;
1692                 }
1693         } else {
1694                 cxgb_setup_msix(sc, sc->msi_count);
1695         }
1696
1697         t3_sge_start(sc);
1698         t3_intr_enable(sc);
1699
1700         if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1701             is_offload(sc) && init_tp_parity(sc) == 0)
1702                 sc->flags |= TP_PARITY_INIT;
1703
1704         if (sc->flags & TP_PARITY_INIT) {
1705                 t3_write_reg(sc, A_TP_INT_CAUSE,
1706                                 F_CMCACHEPERR | F_ARPLUTPERR);
1707                 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1708         }
1709
1710
1711         if (!(sc->flags & QUEUES_BOUND)) {
1712                 bind_qsets(sc);
1713                 sc->flags |= QUEUES_BOUND;
1714         }
1715 out:
1716         return (err);
1717 irq_err:
1718         CH_ERR(sc, "request_irq failed, err %d\n", err);
1719         goto out;
1720 }
1721
1722
1723 /*
1724  * Release resources when all the ports and offloading have been stopped.
1725  */
1726 static void
1727 cxgb_down_locked(struct adapter *sc)
1728 {
1729
1730         t3_sge_stop(sc);
1731         t3_intr_disable(sc);
1732
1733         if (sc->intr_tag != NULL) {
1734                 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1735                 sc->intr_tag = NULL;
1736         }
1737         if (sc->irq_res != NULL) {
1738                 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1739                     sc->irq_rid, sc->irq_res);
1740                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1741                     sc->irq_res);
1742                 sc->irq_res = NULL;
1743         }
1744
1745         if (sc->flags & USING_MSIX)
1746                 cxgb_teardown_msix(sc);
1747
1748         callout_stop(&sc->cxgb_tick_ch);
1749         callout_stop(&sc->sge_timer_ch);
1750         callout_drain(&sc->cxgb_tick_ch);
1751         callout_drain(&sc->sge_timer_ch);
1752
1753         if (sc->tq != NULL) {
1754                 printf("draining slow intr\n");
1755
1756                 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1757                         printf("draining ext intr\n");
1758                 taskqueue_drain(sc->tq, &sc->ext_intr_task);
1759                 printf("draining tick task\n");
1760                 taskqueue_drain(sc->tq, &sc->tick_task);
1761         }
1762         ADAPTER_UNLOCK(sc);
1763 }
1764
1765 static int
1766 offload_open(struct port_info *pi)
1767 {
1768         struct adapter *adapter = pi->adapter;
1769         struct t3cdev *tdev = &adapter->tdev;
1770
1771         int adap_up = adapter->open_device_map & PORT_MASK;
1772         int err = 0;
1773
1774         CTR1(KTR_CXGB, "device_map=0x%x", adapter->open_device_map);
1775         if (atomic_cmpset_int(&adapter->open_device_map,
1776                 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)),
1777                 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0)
1778                 return (0);
1779
1780
1781         if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1782                 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", adapter->open_device_map);
1783         ADAPTER_LOCK(pi->adapter);
1784         if (!adap_up)
1785                 err = cxgb_up(adapter);
1786         ADAPTER_UNLOCK(pi->adapter);
1787         if (err)
1788                 return (err);
1789
1790         t3_tp_set_offload_mode(adapter, 1);
1791         tdev->lldev = pi->ifp;
1792
1793         init_port_mtus(adapter);
1794         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1795                      adapter->params.b_wnd,
1796                      adapter->params.rev == 0 ?
1797                        adapter->port[0].ifp->if_mtu : 0xffff);
1798         init_smt(adapter);
1799         /* Call back all registered clients */
1800         cxgb_add_clients(tdev);
1801
1802         /* restore them in case the offload module has changed them */
1803         if (err) {
1804                 t3_tp_set_offload_mode(adapter, 0);
1805                 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1806                 cxgb_set_dummy_ops(tdev);
1807         }
1808         return (err);
1809 }
1810
1811 static int
1812 offload_close(struct t3cdev *tdev)
1813 {
1814         struct adapter *adapter = tdev2adap(tdev);
1815
1816         if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1817                 return (0);
1818
1819         /* Call back all registered clients */
1820         cxgb_remove_clients(tdev);
1821
1822         tdev->lldev = NULL;
1823         cxgb_set_dummy_ops(tdev);
1824         t3_tp_set_offload_mode(adapter, 0);
1825         clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1826
1827         ADAPTER_LOCK(adapter);
1828         if (!adapter->open_device_map)
1829                 cxgb_down_locked(adapter);
1830         else
1831                 ADAPTER_UNLOCK(adapter);
1832         return (0);
1833 }
1834
1835
1836 static void
1837 cxgb_init(void *arg)
1838 {
1839         struct port_info *p = arg;
1840
1841         PORT_LOCK(p);
1842         cxgb_init_locked(p);
1843         PORT_UNLOCK(p);
1844 }
1845
1846 static void
1847 cxgb_init_locked(struct port_info *p)
1848 {
1849         struct ifnet *ifp;
1850         adapter_t *sc = p->adapter;
1851         int err;
1852
1853         PORT_LOCK_ASSERT_OWNED(p);
1854         ifp = p->ifp;
1855
1856         ADAPTER_LOCK(p->adapter);
1857         if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) {
1858                 ADAPTER_UNLOCK(p->adapter);
1859                 cxgb_stop_locked(p);
1860                 return;
1861         }
1862         if (p->adapter->open_device_map == 0) {
1863                 t3_intr_clear(sc);
1864         }
1865         setbit(&p->adapter->open_device_map, p->port_id);
1866         ADAPTER_UNLOCK(p->adapter);
1867
1868         if (is_offload(sc) && !ofld_disable) {
1869                 err = offload_open(p);
1870                 if (err)
1871                         log(LOG_WARNING,
1872                             "Could not initialize offload capabilities\n");
1873         }
1874 #if !defined(LINK_ATTACH)
1875         cxgb_link_start(p);
1876         t3_link_changed(sc, p->port_id);
1877 #endif
1878         ifp->if_baudrate = p->link_config.speed * 1000000;
1879
1880         device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
1881         t3_port_intr_enable(sc, p->port_id);
1882
1883         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
1884         t3_sge_reset_adapter(sc);
1885
1886         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1887         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1888 }
1889
1890 static void
1891 cxgb_set_rxmode(struct port_info *p)
1892 {
1893         struct t3_rx_mode rm;
1894         struct cmac *mac = &p->mac;
1895
1896         t3_init_rx_mode(&rm, p);
1897         mtx_lock(&p->adapter->mdio_lock);
1898         t3_mac_set_rx_mode(mac, &rm);
1899         mtx_unlock(&p->adapter->mdio_lock);
1900 }
1901
1902 static void
1903 cxgb_stop_locked(struct port_info *pi)
1904 {
1905         struct ifnet *ifp;
1906
1907         PORT_LOCK_ASSERT_OWNED(pi);
1908         ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter);
1909
1910         ifp = pi->ifp;
1911         t3_port_intr_disable(pi->adapter, pi->port_id);
1912         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1913
1914         /* disable pause frames */
1915         t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset,
1916                          F_TXPAUSEEN, 0);
1917
1918         /* Reset RX FIFO HWM */
1919         t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG +  pi->mac.offset,
1920                          V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1921
1922
1923         ADAPTER_LOCK(pi->adapter);
1924         clrbit(&pi->adapter->open_device_map, pi->port_id);
1925
1926         if (pi->adapter->open_device_map == 0) {
1927                 cxgb_down_locked(pi->adapter);
1928         } else
1929                 ADAPTER_UNLOCK(pi->adapter);
1930
1931 #if !defined(LINK_ATTACH)
1932         DELAY(100);
1933
1934         /* Wait for TXFIFO empty */
1935         t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
1936                         F_TXFIFO_EMPTY, 1, 20, 5);
1937
1938         DELAY(100);
1939         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1940
1941         pi->phy.ops->power_down(&pi->phy, 1);
1942 #endif
1943
1944 }
1945
1946 static int
1947 cxgb_set_mtu(struct port_info *p, int mtu)
1948 {
1949         struct ifnet *ifp = p->ifp;
1950         int error = 0;
1951
1952         if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1953                 error = EINVAL;
1954         else if (ifp->if_mtu != mtu) {
1955                 PORT_LOCK(p);
1956                 ifp->if_mtu = mtu;
1957                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1958                         cxgb_stop_locked(p);
1959                         cxgb_init_locked(p);
1960                 }
1961                 PORT_UNLOCK(p);
1962         }
1963         return (error);
1964 }
1965
1966 /*
1967  * Mark lro enabled or disabled in all qsets for this port
1968  */
1969 static int
1970 cxgb_set_lro(struct port_info *p, int enabled)
1971 {
1972         int i;
1973         struct adapter *adp = p->adapter;
1974         struct sge_qset *q;
1975
1976         PORT_LOCK_ASSERT_OWNED(p);
1977         for (i = 0; i < p->nqsets; i++) {
1978                 q = &adp->sge.qs[p->first_qset + i];
1979                 q->lro.enabled = (enabled != 0);
1980         }
1981         return (0);
1982 }
1983
1984 static int
1985 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1986 {
1987         struct port_info *p = ifp->if_softc;
1988         struct ifaddr *ifa = (struct ifaddr *)data;
1989         struct ifreq *ifr = (struct ifreq *)data;
1990         int flags, error = 0, reinit = 0;
1991         uint32_t mask;
1992
1993         /*
1994          * XXX need to check that we aren't in the middle of an unload
1995          */
1996         switch (command) {
1997         case SIOCSIFMTU:
1998                 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1999                 break;
2000         case SIOCSIFADDR:
2001                 if (ifa->ifa_addr->sa_family == AF_INET) {
2002                         ifp->if_flags |= IFF_UP;
2003                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2004                                 PORT_LOCK(p);
2005                                 cxgb_init_locked(p);
2006                                 PORT_UNLOCK(p);
2007                         }
2008                         arp_ifinit(ifp, ifa);
2009                 } else
2010                         error = ether_ioctl(ifp, command, data);
2011                 break;
2012         case SIOCSIFFLAGS:
2013                 PORT_LOCK(p);
2014                 if (ifp->if_flags & IFF_UP) {
2015                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2016                                 flags = p->if_flags;
2017                                 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
2018                                     ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
2019                                         cxgb_set_rxmode(p);
2020                         } else
2021                                 cxgb_init_locked(p);
2022                         p->if_flags = ifp->if_flags;
2023                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2024                         cxgb_stop_locked(p);
2025
2026                 PORT_UNLOCK(p);
2027                 break;
2028         case SIOCADDMULTI:
2029         case SIOCDELMULTI:
2030                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2031                         cxgb_set_rxmode(p);
2032                 }
2033                 break;
2034         case SIOCSIFMEDIA:
2035         case SIOCGIFMEDIA:
2036                 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2037                 break;
2038         case SIOCSIFCAP:
2039                 PORT_LOCK(p);
2040                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2041                 if (mask & IFCAP_TXCSUM) {
2042                         if (IFCAP_TXCSUM & ifp->if_capenable) {
2043                                 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
2044                                 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
2045                                     | CSUM_IP | CSUM_TSO);
2046                         } else {
2047                                 ifp->if_capenable |= IFCAP_TXCSUM;
2048                                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP
2049                                     | CSUM_IP);
2050                         }
2051                 }
2052                 if (mask & IFCAP_RXCSUM) {
2053                         ifp->if_capenable ^= IFCAP_RXCSUM;
2054                 }
2055                 if (mask & IFCAP_TSO4) {
2056                         if (IFCAP_TSO4 & ifp->if_capenable) {
2057                                 ifp->if_capenable &= ~IFCAP_TSO4;
2058                                 ifp->if_hwassist &= ~CSUM_TSO;
2059                         } else if (IFCAP_TXCSUM & ifp->if_capenable) {
2060                                 ifp->if_capenable |= IFCAP_TSO4;
2061                                 ifp->if_hwassist |= CSUM_TSO;
2062                         } else {
2063                                 if (cxgb_debug)
2064                                         printf("cxgb requires tx checksum offload"
2065                                             " be enabled to use TSO\n");
2066                                 error = EINVAL;
2067                         }
2068                 }
2069                 if (mask & IFCAP_LRO) {
2070                         ifp->if_capenable ^= IFCAP_LRO;
2071
2072                         /* Safe to do this even if cxgb_up not called yet */
2073                         cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2074                 }
2075                 if (mask & IFCAP_VLAN_HWTAGGING) {
2076                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2077                         reinit = ifp->if_drv_flags & IFF_DRV_RUNNING;
2078                 }
2079                 if (mask & IFCAP_VLAN_MTU) {
2080                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
2081                         reinit = ifp->if_drv_flags & IFF_DRV_RUNNING;
2082                 }
2083                 if (mask & IFCAP_VLAN_HWCSUM) {
2084                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2085                 }
2086                 if (reinit) {
2087                         cxgb_stop_locked(p);
2088                         cxgb_init_locked(p);
2089                 }
2090                 PORT_UNLOCK(p);
2091
2092 #ifdef VLAN_CAPABILITIES
2093                 VLAN_CAPABILITIES(ifp);
2094 #endif
2095                 break;
2096         default:
2097                 error = ether_ioctl(ifp, command, data);
2098                 break;
2099         }
2100         return (error);
2101 }
2102
2103 static int
2104 cxgb_media_change(struct ifnet *ifp)
2105 {
2106         if_printf(ifp, "media change not supported\n");
2107         return (ENXIO);
2108 }
2109
2110 static void
2111 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2112 {
2113         struct port_info *p = ifp->if_softc;
2114
2115         ifmr->ifm_status = IFM_AVALID;
2116         ifmr->ifm_active = IFM_ETHER;
2117
2118         if (!p->link_config.link_ok)
2119                 return;
2120
2121         ifmr->ifm_status |= IFM_ACTIVE;
2122
2123         switch (p->link_config.speed) {
2124         case 10:
2125                 ifmr->ifm_active |= IFM_10_T;
2126                 break;
2127         case 100:
2128                 ifmr->ifm_active |= IFM_100_TX;
2129                         break;
2130         case 1000:
2131                 ifmr->ifm_active |= IFM_1000_T;
2132                 break;
2133         }
2134
2135         if (p->link_config.duplex)
2136                 ifmr->ifm_active |= IFM_FDX;
2137         else
2138                 ifmr->ifm_active |= IFM_HDX;
2139 }
2140
2141 static void
2142 cxgb_async_intr(void *data)
2143 {
2144         adapter_t *sc = data;
2145
2146         if (cxgb_debug)
2147                 device_printf(sc->dev, "cxgb_async_intr\n");
2148         /*
2149          * May need to sleep - defer to taskqueue
2150          */
2151         taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2152 }
2153
2154 static void
2155 cxgb_ext_intr_handler(void *arg, int count)
2156 {
2157         adapter_t *sc = (adapter_t *)arg;
2158
2159         if (cxgb_debug)
2160                 printf("cxgb_ext_intr_handler\n");
2161
2162         t3_phy_intr_handler(sc);
2163
2164         /* Now reenable external interrupts */
2165         ADAPTER_LOCK(sc);
2166         if (sc->slow_intr_mask) {
2167                 sc->slow_intr_mask |= F_T3DBG;
2168                 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
2169                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
2170         }
2171         ADAPTER_UNLOCK(sc);
2172 }
2173
2174 static void
2175 check_link_status(adapter_t *sc)
2176 {
2177         int i;
2178
2179         for (i = 0; i < (sc)->params.nports; ++i) {
2180                 struct port_info *p = &sc->port[i];
2181
2182                 if (!(p->phy.caps & SUPPORTED_IRQ))
2183                         t3_link_changed(sc, i);
2184                 p->ifp->if_baudrate = p->link_config.speed * 1000000;
2185         }
2186 }
2187
2188 static void
2189 check_t3b2_mac(struct adapter *adapter)
2190 {
2191         int i;
2192
2193         if(adapter->flags & CXGB_SHUTDOWN)
2194                 return;
2195
2196         for_each_port(adapter, i) {
2197                 struct port_info *p = &adapter->port[i];
2198                 struct ifnet *ifp = p->ifp;
2199                 int status;
2200
2201                 if(adapter->flags & CXGB_SHUTDOWN)
2202                         return;
2203
2204                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2205                         continue;
2206
2207                 status = 0;
2208                 PORT_LOCK(p);
2209                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
2210                         status = t3b2_mac_watchdog_task(&p->mac);
2211                 if (status == 1)
2212                         p->mac.stats.num_toggled++;
2213                 else if (status == 2) {
2214                         struct cmac *mac = &p->mac;
2215                         int mtu = ifp->if_mtu;
2216
2217                         if (ifp->if_capenable & IFCAP_VLAN_MTU)
2218                                 mtu += ETHER_VLAN_ENCAP_LEN;
2219                         t3_mac_set_mtu(mac, mtu);
2220                         t3_mac_set_address(mac, 0, p->hw_addr);
2221                         cxgb_set_rxmode(p);
2222                         t3_link_start(&p->phy, mac, &p->link_config);
2223                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2224                         t3_port_intr_enable(adapter, p->port_id);
2225                         p->mac.stats.num_resets++;
2226                 }
2227                 PORT_UNLOCK(p);
2228         }
2229 }
2230
2231 static void
2232 cxgb_tick(void *arg)
2233 {
2234         adapter_t *sc = (adapter_t *)arg;
2235
2236         if(sc->flags & CXGB_SHUTDOWN)
2237                 return;
2238
2239         taskqueue_enqueue(sc->tq, &sc->tick_task);
2240         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
2241 }
2242
2243 static void
2244 cxgb_tick_handler(void *arg, int count)
2245 {
2246         adapter_t *sc = (adapter_t *)arg;
2247         const struct adapter_params *p = &sc->params;
2248         int i;
2249
2250         if(sc->flags & CXGB_SHUTDOWN)
2251                 return;
2252
2253         ADAPTER_LOCK(sc);
2254         if (p->linkpoll_period)
2255                 check_link_status(sc);
2256
2257         sc->check_task_cnt++;
2258
2259         /*
2260          * adapter lock can currently only be acquired after the
2261          * port lock
2262          */
2263         ADAPTER_UNLOCK(sc);
2264
2265         if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2266                 check_t3b2_mac(sc);
2267
2268         /* Update MAC stats if it's time to do so */
2269         if (!p->linkpoll_period ||
2270             (sc->check_task_cnt * p->linkpoll_period) / 10 >=
2271             p->stats_update_period) {
2272                 for_each_port(sc, i) {
2273                         struct port_info *port = &sc->port[i];
2274                         PORT_LOCK(port);
2275                         t3_mac_update_stats(&port->mac);
2276                         PORT_UNLOCK(port);
2277                 }
2278                 sc->check_task_cnt = 0;
2279         }
2280 }
2281
2282 static void
2283 touch_bars(device_t dev)
2284 {
2285         /*
2286          * Don't enable yet
2287          */
2288 #if !defined(__LP64__) && 0
2289         u32 v;
2290
2291         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2292         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2293         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2294         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2295         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2296         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2297 #endif
2298 }
2299
2300 static int
2301 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2302 {
2303         uint8_t *buf;
2304         int err = 0;
2305         u32 aligned_offset, aligned_len, *p;
2306         struct adapter *adapter = pi->adapter;
2307
2308
2309         aligned_offset = offset & ~3;
2310         aligned_len = (len + (offset & 3) + 3) & ~3;
2311
2312         if (aligned_offset != offset || aligned_len != len) {
2313                 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2314                 if (!buf)
2315                         return (ENOMEM);
2316                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2317                 if (!err && aligned_len > 4)
2318                         err = t3_seeprom_read(adapter,
2319                                               aligned_offset + aligned_len - 4,
2320                                               (u32 *)&buf[aligned_len - 4]);
2321                 if (err)
2322                         goto out;
2323                 memcpy(buf + (offset & 3), data, len);
2324         } else
2325                 buf = (uint8_t *)(uintptr_t)data;
2326
2327         err = t3_seeprom_wp(adapter, 0);
2328         if (err)
2329                 goto out;
2330
2331         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2332                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2333                 aligned_offset += 4;
2334         }
2335
2336         if (!err)
2337                 err = t3_seeprom_wp(adapter, 1);
2338 out:
2339         if (buf != data)
2340                 free(buf, M_DEVBUF);
2341         return err;
2342 }
2343
2344
2345 static int
2346 in_range(int val, int lo, int hi)
2347 {
2348         return val < 0 || (val <= hi && val >= lo);
2349 }
2350
2351 static int
2352 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
2353 {
2354        return (0);
2355 }
2356
2357 static int
2358 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
2359 {
2360        return (0);
2361 }
2362
2363 static int
2364 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2365     int fflag, struct thread *td)
2366 {
2367         int mmd, error = 0;
2368         struct port_info *pi = dev->si_drv1;
2369         adapter_t *sc = pi->adapter;
2370
2371 #ifdef PRIV_SUPPORTED
2372         if (priv_check(td, PRIV_DRIVER)) {
2373                 if (cxgb_debug)
2374                         printf("user does not have access to privileged ioctls\n");
2375                 return (EPERM);
2376         }
2377 #else
2378         if (suser(td)) {
2379                 if (cxgb_debug)
2380                         printf("user does not have access to privileged ioctls\n");
2381                 return (EPERM);
2382         }
2383 #endif
2384
2385         switch (cmd) {
2386         case CHELSIO_GET_MIIREG: {
2387                 uint32_t val;
2388                 struct cphy *phy = &pi->phy;
2389                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2390
2391                 if (!phy->mdio_read)
2392                         return (EOPNOTSUPP);
2393                 if (is_10G(sc)) {
2394                         mmd = mid->phy_id >> 8;
2395                         if (!mmd)
2396                                 mmd = MDIO_DEV_PCS;
2397                         else if (mmd > MDIO_DEV_XGXS)
2398                                 return (EINVAL);
2399
2400                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2401                                              mid->reg_num, &val);
2402                 } else
2403                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2404                                              mid->reg_num & 0x1f, &val);
2405                 if (error == 0)
2406                         mid->val_out = val;
2407                 break;
2408         }
2409         case CHELSIO_SET_MIIREG: {
2410                 struct cphy *phy = &pi->phy;
2411                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2412
2413                 if (!phy->mdio_write)
2414                         return (EOPNOTSUPP);
2415                 if (is_10G(sc)) {
2416                         mmd = mid->phy_id >> 8;
2417                         if (!mmd)
2418                                 mmd = MDIO_DEV_PCS;
2419                         else if (mmd > MDIO_DEV_XGXS)
2420                                 return (EINVAL);
2421
2422                         error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2423                                               mmd, mid->reg_num, mid->val_in);
2424                 } else
2425                         error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2426                                               mid->reg_num & 0x1f,
2427                                               mid->val_in);
2428                 break;
2429         }
2430         case CHELSIO_SETREG: {
2431                 struct ch_reg *edata = (struct ch_reg *)data;
2432                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2433                         return (EFAULT);
2434                 t3_write_reg(sc, edata->addr, edata->val);
2435                 break;
2436         }
2437         case CHELSIO_GETREG: {
2438                 struct ch_reg *edata = (struct ch_reg *)data;
2439                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2440                         return (EFAULT);
2441                 edata->val = t3_read_reg(sc, edata->addr);
2442                 break;
2443         }
2444         case CHELSIO_GET_SGE_CONTEXT: {
2445                 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2446                 mtx_lock_spin(&sc->sge.reg_lock);
2447                 switch (ecntxt->cntxt_type) {
2448                 case CNTXT_TYPE_EGRESS:
2449                         error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2450                             ecntxt->data);
2451                         break;
2452                 case CNTXT_TYPE_FL:
2453                         error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2454                             ecntxt->data);
2455                         break;
2456                 case CNTXT_TYPE_RSP:
2457                         error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2458                             ecntxt->data);
2459                         break;
2460                 case CNTXT_TYPE_CQ:
2461                         error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2462                             ecntxt->data);
2463                         break;
2464                 default:
2465                         error = EINVAL;
2466                         break;
2467                 }
2468                 mtx_unlock_spin(&sc->sge.reg_lock);
2469                 break;
2470         }
2471         case CHELSIO_GET_SGE_DESC: {
2472                 struct ch_desc *edesc = (struct ch_desc *)data;
2473                 int ret;
2474                 if (edesc->queue_num >= SGE_QSETS * 6)
2475                         return (EINVAL);
2476                 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2477                     edesc->queue_num % 6, edesc->idx, edesc->data);
2478                 if (ret < 0)
2479                         return (EINVAL);
2480                 edesc->size = ret;
2481                 break;
2482         }
2483         case CHELSIO_GET_QSET_PARAMS: {
2484                 struct qset_params *q;
2485                 struct ch_qset_params *t = (struct ch_qset_params *)data;
2486                 int q1 = pi->first_qset;
2487                 int nqsets = pi->nqsets;
2488                 int i;
2489
2490                 if (t->qset_idx >= nqsets)
2491                         return EINVAL;
2492
2493                 i = q1 + t->qset_idx;
2494                 q = &sc->params.sge.qset[i];
2495                 t->rspq_size   = q->rspq_size;
2496                 t->txq_size[0] = q->txq_size[0];
2497                 t->txq_size[1] = q->txq_size[1];
2498                 t->txq_size[2] = q->txq_size[2];
2499                 t->fl_size[0]  = q->fl_size;
2500                 t->fl_size[1]  = q->jumbo_size;
2501                 t->polling     = q->polling;
2502                 t->lro         = q->lro;
2503                 t->intr_lat    = q->coalesce_usecs;
2504                 t->cong_thres  = q->cong_thres;
2505                 t->qnum        = i;
2506
2507                 if (sc->flags & USING_MSIX)
2508                         t->vector = rman_get_start(sc->msix_irq_res[i]);
2509                 else
2510                         t->vector = rman_get_start(sc->irq_res);
2511
2512                 break;
2513         }
2514         case CHELSIO_GET_QSET_NUM: {
2515                 struct ch_reg *edata = (struct ch_reg *)data;
2516                 edata->val = pi->nqsets;
2517                 break;
2518         }
2519         case CHELSIO_LOAD_FW: {
2520                 uint8_t *fw_data;
2521                 uint32_t vers;
2522                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2523
2524                 /*
2525                  * You're allowed to load a firmware only before FULL_INIT_DONE
2526                  *
2527                  * FW_UPTODATE is also set so the rest of the initialization
2528                  * will not overwrite what was loaded here.  This gives you the
2529                  * flexibility to load any firmware (and maybe shoot yourself in
2530                  * the foot).
2531                  */
2532
2533                 ADAPTER_LOCK(sc);
2534                 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2535                         ADAPTER_UNLOCK(sc);
2536                         return (EBUSY);
2537                 }
2538
2539                 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2540                 if (!fw_data)
2541                         error = ENOMEM;
2542                 else
2543                         error = copyin(t->buf, fw_data, t->len);
2544
2545                 if (!error)
2546                         error = -t3_load_fw(sc, fw_data, t->len);
2547
2548                 if (t3_get_fw_version(sc, &vers) == 0) {
2549                         snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2550                             "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2551                             G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2552                 }
2553
2554                 if (!error)
2555                         sc->flags |= FW_UPTODATE;
2556
2557                 free(fw_data, M_DEVBUF);
2558                 ADAPTER_UNLOCK(sc);
2559                 break;
2560         }
2561         case CHELSIO_LOAD_BOOT: {
2562                 uint8_t *boot_data;
2563                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2564
2565                 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2566                 if (!boot_data)
2567                         return ENOMEM;
2568
2569                 error = copyin(t->buf, boot_data, t->len);
2570                 if (!error)
2571                         error = -t3_load_boot(sc, boot_data, t->len);
2572
2573                 free(boot_data, M_DEVBUF);
2574                 break;
2575         }
2576         case CHELSIO_GET_PM: {
2577                 struct ch_pm *m = (struct ch_pm *)data;
2578                 struct tp_params *p = &sc->params.tp;
2579
2580                 if (!is_offload(sc))
2581                         return (EOPNOTSUPP);
2582
2583                 m->tx_pg_sz = p->tx_pg_size;
2584                 m->tx_num_pg = p->tx_num_pgs;
2585                 m->rx_pg_sz  = p->rx_pg_size;
2586                 m->rx_num_pg = p->rx_num_pgs;
2587                 m->pm_total  = p->pmtx_size + p->chan_rx_size * p->nchan;
2588
2589                 break;
2590         }
2591         case CHELSIO_SET_PM: {
2592                 struct ch_pm *m = (struct ch_pm *)data;
2593                 struct tp_params *p = &sc->params.tp;
2594
2595                 if (!is_offload(sc))
2596                         return (EOPNOTSUPP);
2597                 if (sc->flags & FULL_INIT_DONE)
2598                         return (EBUSY);
2599
2600                 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2601                     !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2602                         return (EINVAL);        /* not power of 2 */
2603                 if (!(m->rx_pg_sz & 0x14000))
2604                         return (EINVAL);        /* not 16KB or 64KB */
2605                 if (!(m->tx_pg_sz & 0x1554000))
2606                         return (EINVAL);
2607                 if (m->tx_num_pg == -1)
2608                         m->tx_num_pg = p->tx_num_pgs;
2609                 if (m->rx_num_pg == -1)
2610                         m->rx_num_pg = p->rx_num_pgs;
2611                 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2612                         return (EINVAL);
2613                 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2614                     m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2615                         return (EINVAL);
2616
2617                 p->rx_pg_size = m->rx_pg_sz;
2618                 p->tx_pg_size = m->tx_pg_sz;
2619                 p->rx_num_pgs = m->rx_num_pg;
2620                 p->tx_num_pgs = m->tx_num_pg;
2621                 break;
2622         }
2623         case CHELSIO_SETMTUTAB: {
2624                 struct ch_mtus *m = (struct ch_mtus *)data;
2625                 int i;
2626
2627                 if (!is_offload(sc))
2628                         return (EOPNOTSUPP);
2629                 if (offload_running(sc))
2630                         return (EBUSY);
2631                 if (m->nmtus != NMTUS)
2632                         return (EINVAL);
2633                 if (m->mtus[0] < 81)         /* accommodate SACK */
2634                         return (EINVAL);
2635
2636                 /*
2637                  * MTUs must be in ascending order
2638                  */
2639                 for (i = 1; i < NMTUS; ++i)
2640                         if (m->mtus[i] < m->mtus[i - 1])
2641                                 return (EINVAL);
2642
2643                 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2644                 break;
2645         }
2646         case CHELSIO_GETMTUTAB: {
2647                 struct ch_mtus *m = (struct ch_mtus *)data;
2648
2649                 if (!is_offload(sc))
2650                         return (EOPNOTSUPP);
2651
2652                 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2653                 m->nmtus = NMTUS;
2654                 break;
2655         }
2656         case CHELSIO_GET_MEM: {
2657                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2658                 struct mc7 *mem;
2659                 uint8_t *useraddr;
2660                 u64 buf[32];
2661
2662                 /*
2663                  * Use these to avoid modifying len/addr in the the return
2664                  * struct
2665                  */
2666                 uint32_t len = t->len, addr = t->addr;
2667
2668                 if (!is_offload(sc))
2669                         return (EOPNOTSUPP);
2670                 if (!(sc->flags & FULL_INIT_DONE))
2671                         return (EIO);         /* need the memory controllers */
2672                 if ((addr & 0x7) || (len & 0x7))
2673                         return (EINVAL);
2674                 if (t->mem_id == MEM_CM)
2675                         mem = &sc->cm;
2676                 else if (t->mem_id == MEM_PMRX)
2677                         mem = &sc->pmrx;
2678                 else if (t->mem_id == MEM_PMTX)
2679                         mem = &sc->pmtx;
2680                 else
2681                         return (EINVAL);
2682
2683                 /*
2684                  * Version scheme:
2685                  * bits 0..9: chip version
2686                  * bits 10..15: chip revision
2687                  */
2688                 t->version = 3 | (sc->params.rev << 10);
2689
2690                 /*
2691                  * Read 256 bytes at a time as len can be large and we don't
2692                  * want to use huge intermediate buffers.
2693                  */
2694                 useraddr = (uint8_t *)t->buf;
2695                 while (len) {
2696                         unsigned int chunk = min(len, sizeof(buf));
2697
2698                         error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2699                         if (error)
2700                                 return (-error);
2701                         if (copyout(buf, useraddr, chunk))
2702                                 return (EFAULT);
2703                         useraddr += chunk;
2704                         addr += chunk;
2705                         len -= chunk;
2706                 }
2707                 break;
2708         }
2709         case CHELSIO_READ_TCAM_WORD: {
2710                 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2711
2712                 if (!is_offload(sc))
2713                         return (EOPNOTSUPP);
2714                 if (!(sc->flags & FULL_INIT_DONE))
2715                         return (EIO);         /* need MC5 */
2716                 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2717                 break;
2718         }
2719         case CHELSIO_SET_TRACE_FILTER: {
2720                 struct ch_trace *t = (struct ch_trace *)data;
2721                 const struct trace_params *tp;
2722
2723                 tp = (const struct trace_params *)&t->sip;
2724                 if (t->config_tx)
2725                         t3_config_trace_filter(sc, tp, 0, t->invert_match,
2726                                                t->trace_tx);
2727                 if (t->config_rx)
2728                         t3_config_trace_filter(sc, tp, 1, t->invert_match,
2729                                                t->trace_rx);
2730                 break;
2731         }
2732         case CHELSIO_SET_PKTSCHED: {
2733                 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2734                 if (sc->open_device_map == 0)
2735                         return (EAGAIN);
2736                 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2737                     p->binding);
2738                 break;
2739         }
2740         case CHELSIO_IFCONF_GETREGS: {
2741                 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2742                 int reglen = cxgb_get_regs_len();
2743                 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2744                 if (buf == NULL) {
2745                         return (ENOMEM);
2746                 }
2747                 if (regs->len > reglen)
2748                         regs->len = reglen;
2749                 else if (regs->len < reglen)
2750                         error = E2BIG;
2751
2752                 if (!error) {
2753                         cxgb_get_regs(sc, regs, buf);
2754                         error = copyout(buf, regs->data, reglen);
2755                 }
2756                 free(buf, M_DEVBUF);
2757
2758                 break;
2759         }
2760         case CHELSIO_SET_HW_SCHED: {
2761                 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2762                 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2763
2764                 if ((sc->flags & FULL_INIT_DONE) == 0)
2765                         return (EAGAIN);       /* need TP to be initialized */
2766                 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2767                     !in_range(t->channel, 0, 1) ||
2768                     !in_range(t->kbps, 0, 10000000) ||
2769                     !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2770                     !in_range(t->flow_ipg, 0,
2771                               dack_ticks_to_usec(sc, 0x7ff)))
2772                         return (EINVAL);
2773
2774                 if (t->kbps >= 0) {
2775                         error = t3_config_sched(sc, t->kbps, t->sched);
2776                         if (error < 0)
2777                                 return (-error);
2778                 }
2779                 if (t->class_ipg >= 0)
2780                         t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2781                 if (t->flow_ipg >= 0) {
2782                         t->flow_ipg *= 1000;     /* us -> ns */
2783                         t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2784                 }
2785                 if (t->mode >= 0) {
2786                         int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2787
2788                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2789                                          bit, t->mode ? bit : 0);
2790                 }
2791                 if (t->channel >= 0)
2792                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2793                                          1 << t->sched, t->channel << t->sched);
2794                 break;
2795         }
2796         case CHELSIO_GET_EEPROM: {
2797                 int i;
2798                 struct ch_eeprom *e = (struct ch_eeprom *)data;
2799                 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2800
2801                 if (buf == NULL) {
2802                         return (ENOMEM);
2803                 }
2804                 e->magic = EEPROM_MAGIC;
2805                 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2806                         error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2807
2808                 if (!error)
2809                         error = copyout(buf + e->offset, e->data, e->len);
2810
2811                 free(buf, M_DEVBUF);
2812                 break;
2813         }
2814         case CHELSIO_CLEAR_STATS: {
2815                 if (!(sc->flags & FULL_INIT_DONE))
2816                         return EAGAIN;
2817
2818                 PORT_LOCK(pi);
2819                 t3_mac_update_stats(&pi->mac);
2820                 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
2821                 PORT_UNLOCK(pi);
2822                 break;
2823         }
2824         default:
2825                 return (EOPNOTSUPP);
2826                 break;
2827         }
2828
2829         return (error);
2830 }
2831
2832 static __inline void
2833 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2834     unsigned int end)
2835 {
2836         uint32_t *p = (uint32_t *)(buf + start);
2837
2838         for ( ; start <= end; start += sizeof(uint32_t))
2839                 *p++ = t3_read_reg(ap, start);
2840 }
2841
2842 #define T3_REGMAP_SIZE (3 * 1024)
2843 static int
2844 cxgb_get_regs_len(void)
2845 {
2846         return T3_REGMAP_SIZE;
2847 }
2848
2849 static void
2850 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
2851 {
2852
2853         /*
2854          * Version scheme:
2855          * bits 0..9: chip version
2856          * bits 10..15: chip revision
2857          * bit 31: set for PCIe cards
2858          */
2859         regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2860
2861         /*
2862          * We skip the MAC statistics registers because they are clear-on-read.
2863          * Also reading multi-register stats would need to synchronize with the
2864          * periodic mac stats accumulation.  Hard to justify the complexity.
2865          */
2866         memset(buf, 0, cxgb_get_regs_len());
2867         reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2868         reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2869         reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2870         reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2871         reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2872         reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2873                        XGM_REG(A_XGM_SERDES_STAT3, 1));
2874         reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2875                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2876 }
2877
2878
2879 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1);