sys/dev/cxgb/cxgb_main.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2007-2009, Chelsio Inc.
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Neither the name of the Chelsio Corporation nor the names of its
  13     contributors may be used to endorse or promote products derived from
  14     this software without specific prior written permission.
  15
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26 POSSIBILITY OF SUCH DAMAGE.
  27
  28 ***************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/ktr.h>
  44 #include <sys/rman.h>
  45 #include <sys/ioccom.h>
  46 #include <sys/mbuf.h>
  47 #include <sys/linker.h>
  48 #include <sys/firmware.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54 #include <sys/queue.h>
  55 #include <sys/taskqueue.h>
  56 #include <sys/proc.h>
  57
  58 #include <net/bpf.h>
  59 #include <net/ethernet.h>
  60 #include <net/if.h>
  61 #include <net/if_arp.h>
  62 #include <net/if_dl.h>
  63 #include <net/if_media.h>
  64 #include <net/if_types.h>
  65 #include <net/if_vlan_var.h>
  66
  67 #include <netinet/in_systm.h>
  68 #include <netinet/in.h>
  69 #include <netinet/if_ether.h>
  70 #include <netinet/ip.h>
  71 #include <netinet/ip.h>
  72 #include <netinet/tcp.h>
  73 #include <netinet/udp.h>
  74
  75 #include <dev/pci/pcireg.h>
  76 #include <dev/pci/pcivar.h>
  77 #include <dev/pci/pci_private.h>
  78
  79 #include <cxgb_include.h>
  80
  81 #ifdef PRIV_SUPPORTED
  82 #include <sys/priv.h>
  83 #endif
  84
  85 static int cxgb_setup_interrupts(adapter_t *);
  86 static void cxgb_teardown_interrupts(adapter_t *);
  87 static void cxgb_init(void *);
  88 static int cxgb_init_locked(struct port_info *);
  89 static int cxgb_uninit_locked(struct port_info *);
  90 static int cxgb_uninit_synchronized(struct port_info *);
  91 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
  92 static int cxgb_media_change(struct ifnet *);
  93 static int cxgb_ifm_type(int);
  94 static void cxgb_build_medialist(struct port_info *);
  95 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
  96 static int setup_sge_qsets(adapter_t *);
  97 static void cxgb_async_intr(void *);
  98 static void cxgb_ext_intr_handler(void *, int);
  99 static void cxgb_tick_handler(void *, int);
 100 static void cxgb_tick(void *);
 101 static void setup_rss(adapter_t *sc);
 102
 103 /* Attachment glue for the PCI controller end of the device.  Each port of
 104  * the device is attached separately, as defined later.
 105  */
 106 static int cxgb_controller_probe(device_t);
 107 static int cxgb_controller_attach(device_t);
 108 static int cxgb_controller_detach(device_t);
 109 static void cxgb_free(struct adapter *);
 110 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
 111     unsigned int end);
 112 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
 113 static int cxgb_get_regs_len(void);
 114 static int offload_open(struct port_info *pi);
 115 static void touch_bars(device_t dev);
 116 static int offload_close(struct t3cdev *tdev);
 117 static void cxgb_update_mac_settings(struct port_info *p);
 118
 119 static device_method_t cxgb_controller_methods[] = {
 120         DEVMETHOD(device_probe,         cxgb_controller_probe),
 121         DEVMETHOD(device_attach,        cxgb_controller_attach),
 122         DEVMETHOD(device_detach,        cxgb_controller_detach),
 123
 124         /* bus interface */
 125         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 126         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 127
 128         { 0, 0 }
 129 };
 130
 131 static driver_t cxgb_controller_driver = {
 132         "cxgbc",
 133         cxgb_controller_methods,
 134         sizeof(struct adapter)
 135 };
 136
 137 static devclass_t       cxgb_controller_devclass;
 138 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
 139
 140 /*
 141  * Attachment glue for the ports.  Attachment is done directly to the
 142  * controller device.
 143  */
 144 static int cxgb_port_probe(device_t);
 145 static int cxgb_port_attach(device_t);
 146 static int cxgb_port_detach(device_t);
 147
 148 static device_method_t cxgb_port_methods[] = {
 149         DEVMETHOD(device_probe,         cxgb_port_probe),
 150         DEVMETHOD(device_attach,        cxgb_port_attach),
 151         DEVMETHOD(device_detach,        cxgb_port_detach),
 152         { 0, 0 }
 153 };
 154
 155 static driver_t cxgb_port_driver = {
 156         "cxgb",
 157         cxgb_port_methods,
 158         0
 159 };
 160
 161 static d_ioctl_t cxgb_extension_ioctl;
 162 static d_open_t cxgb_extension_open;
 163 static d_close_t cxgb_extension_close;
 164
 165 static struct cdevsw cxgb_cdevsw = {
 166        .d_version =    D_VERSION,
 167        .d_flags =      0,
 168        .d_open =       cxgb_extension_open,
 169        .d_close =      cxgb_extension_close,
 170        .d_ioctl =      cxgb_extension_ioctl,
 171        .d_name =       "cxgb",
 172 };
 173
 174 static devclass_t       cxgb_port_devclass;
 175 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
 176
 177 /*
 178  * The driver uses the best interrupt scheme available on a platform in the
 179  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 180  * of these schemes the driver may consider as follows:
 181  *
 182  * msi = 2: choose from among all three options
 183  * msi = 1 : only consider MSI and pin interrupts
 184  * msi = 0: force pin interrupts
 185  */
 186 static int msi_allowed = 2;
 187
 188 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
 189 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
 190 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
 191     "MSI-X, MSI, INTx selector");
 192
 193 /*
 194  * The driver enables offload as a default.
 195  * To disable it, use ofld_disable = 1.
 196  */
 197 static int ofld_disable = 0;
 198 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
 199 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
 200     "disable ULP offload");
 201
 202 /*
 203  * The driver uses an auto-queue algorithm by default.
 204  * To disable it and force a single queue-set per port, use multiq = 0
 205  */
 206 static int multiq = 1;
 207 TUNABLE_INT("hw.cxgb.multiq", &multiq);
 208 SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
 209     "use min(ncpus/ports, 8) queue-sets per port");
 210
 211 /*
 212  * By default the driver will not update the firmware unless
 213  * it was compiled against a newer version
 214  *
 215  */
 216 static int force_fw_update = 0;
 217 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
 218 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
 219     "update firmware even if up to date");
 220
 221 int cxgb_use_16k_clusters = 1;
 222 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
 223 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
 224     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
 225
 226 /*
 227  * Tune the size of the output queue.
 228  */
 229 int cxgb_snd_queue_len = IFQ_MAXLEN;
 230 TUNABLE_INT("hw.cxgb.snd_queue_len", &cxgb_snd_queue_len);
 231 SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_queue_len, CTLFLAG_RDTUN,
 232     &cxgb_snd_queue_len, 0, "send queue size ");
 233
 234
 235 enum {
 236         MAX_TXQ_ENTRIES      = 16384,
 237         MAX_CTRL_TXQ_ENTRIES = 1024,
 238         MAX_RSPQ_ENTRIES     = 16384,
 239         MAX_RX_BUFFERS       = 16384,
 240         MAX_RX_JUMBO_BUFFERS = 16384,
 241         MIN_TXQ_ENTRIES      = 4,
 242         MIN_CTRL_TXQ_ENTRIES = 4,
 243         MIN_RSPQ_ENTRIES     = 32,
 244         MIN_FL_ENTRIES       = 32,
 245         MIN_FL_JUMBO_ENTRIES = 32
 246 };
 247
 248 struct filter_info {
 249         u32 sip;
 250         u32 sip_mask;
 251         u32 dip;
 252         u16 sport;
 253         u16 dport;
 254         u32 vlan:12;
 255         u32 vlan_prio:3;
 256         u32 mac_hit:1;
 257         u32 mac_idx:4;
 258         u32 mac_vld:1;
 259         u32 pkt_type:2;
 260         u32 report_filter_id:1;
 261         u32 pass:1;
 262         u32 rss:1;
 263         u32 qset:3;
 264         u32 locked:1;
 265         u32 valid:1;
 266 };
 267
 268 enum { FILTER_NO_VLAN_PRI = 7 };
 269
 270 #define EEPROM_MAGIC 0x38E2F10C
 271
 272 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
 273
 274 /* Table for probing the cards.  The desc field isn't actually used */
 275 struct cxgb_ident {
 276         uint16_t        vendor;
 277         uint16_t        device;
 278         int             index;
 279         char            *desc;
 280 } cxgb_identifiers[] = {
 281         {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
 282         {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
 283         {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
 284         {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
 285         {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
 286         {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
 287         {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
 288         {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
 289         {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
 290         {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
 291         {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
 292         {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
 293         {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
 294         {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
 295         {0, 0, 0, NULL}
 296 };
 297
 298 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
 299
 300
 301 static __inline char
 302 t3rev2char(struct adapter *adapter)
 303 {
 304         char rev = 'z';
 305
 306         switch(adapter->params.rev) {
 307         case T3_REV_A:
 308                 rev = 'a';
 309                 break;
 310         case T3_REV_B:
 311         case T3_REV_B2:
 312                 rev = 'b';
 313                 break;
 314         case T3_REV_C:
 315                 rev = 'c';
 316                 break;
 317         }
 318         return rev;
 319 }
 320
 321 static struct cxgb_ident *
 322 cxgb_get_ident(device_t dev)
 323 {
 324         struct cxgb_ident *id;
 325
 326         for (id = cxgb_identifiers; id->desc != NULL; id++) {
 327                 if ((id->vendor == pci_get_vendor(dev)) &&
 328                     (id->device == pci_get_device(dev))) {
 329                         return (id);
 330                 }
 331         }
 332         return (NULL);
 333 }
 334
 335 static const struct adapter_info *
 336 cxgb_get_adapter_info(device_t dev)
 337 {
 338         struct cxgb_ident *id;
 339         const struct adapter_info *ai;
 340
 341         id = cxgb_get_ident(dev);
 342         if (id == NULL)
 343                 return (NULL);
 344
 345         ai = t3_get_adapter_info(id->index);
 346
 347         return (ai);
 348 }
 349
 350 static int
 351 cxgb_controller_probe(device_t dev)
 352 {
 353         const struct adapter_info *ai;
 354         char *ports, buf[80];
 355         int nports;
 356
 357         ai = cxgb_get_adapter_info(dev);
 358         if (ai == NULL)
 359                 return (ENXIO);
 360
 361         nports = ai->nports0 + ai->nports1;
 362         if (nports == 1)
 363                 ports = "port";
 364         else
 365                 ports = "ports";
 366
 367         snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
 368         device_set_desc_copy(dev, buf);
 369         return (BUS_PROBE_DEFAULT);
 370 }
 371
 372 #define FW_FNAME "cxgb_t3fw"
 373 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
 374 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
 375
 376 static int
 377 upgrade_fw(adapter_t *sc)
 378 {
 379 #ifdef FIRMWARE_LATEST
 380         const struct firmware *fw;
 381 #else
 382         struct firmware *fw;
 383 #endif
 384         int status;
 385
 386         if ((fw = firmware_get(FW_FNAME)) == NULL)  {
 387                 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
 388                 return (ENOENT);
 389         } else
 390                 device_printf(sc->dev, "updating firmware on card\n");
 391         status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
 392
 393         device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status);
 394
 395         firmware_put(fw, FIRMWARE_UNLOAD);
 396
 397         return (status);
 398 }
 399
 400 /*
 401  * The cxgb_controller_attach function is responsible for the initial
 402  * bringup of the device.  Its responsibilities include:
 403  *
 404  *  1. Determine if the device supports MSI or MSI-X.
 405  *  2. Allocate bus resources so that we can access the Base Address Register
 406  *  3. Create and initialize mutexes for the controller and its control
 407  *     logic such as SGE and MDIO.
 408  *  4. Call hardware specific setup routine for the adapter as a whole.
 409  *  5. Allocate the BAR for doing MSI-X.
 410  *  6. Setup the line interrupt iff MSI-X is not supported.
 411  *  7. Create the driver's taskq.
 412  *  8. Start one task queue service thread.
 413  *  9. Check if the firmware and SRAM are up-to-date.  They will be
 414  *     auto-updated later (before FULL_INIT_DONE), if required.
 415  * 10. Create a child device for each MAC (port)
 416  * 11. Initialize T3 private state.
 417  * 12. Trigger the LED
 418  * 13. Setup offload iff supported.
 419  * 14. Reset/restart the tick callout.
 420  * 15. Attach sysctls
 421  *
 422  * NOTE: Any modification or deviation from this list MUST be reflected in
 423  * the above comment.  Failure to do so will result in problems on various
 424  * error conditions including link flapping.
 425  */
 426 static int
 427 cxgb_controller_attach(device_t dev)
 428 {
 429         device_t child;
 430         const struct adapter_info *ai;
 431         struct adapter *sc;
 432         int i, error = 0;
 433         uint32_t vers;
 434         int port_qsets = 1;
 435 #ifdef MSI_SUPPORTED
 436         int msi_needed, reg;
 437 #endif
 438         char buf[80];
 439
 440         sc = device_get_softc(dev);
 441         sc->dev = dev;
 442         sc->msi_count = 0;
 443         ai = cxgb_get_adapter_info(dev);
 444
 445         /*
 446          * XXX not really related but a recent addition
 447          */
 448 #ifdef MSI_SUPPORTED
 449         /* find the PCIe link width and set max read request to 4KB*/
 450         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 451                 uint16_t lnk, pectl;
 452                 lnk = pci_read_config(dev, reg + 0x12, 2);
 453                 sc->link_width = (lnk >> 4) & 0x3f;
 454
 455                 pectl = pci_read_config(dev, reg + 0x8, 2);
 456                 pectl = (pectl & ~0x7000) | (5 << 12);
 457                 pci_write_config(dev, reg + 0x8, pectl, 2);
 458         }
 459
 460         if (sc->link_width != 0 && sc->link_width <= 4 &&
 461             (ai->nports0 + ai->nports1) <= 2) {
 462                 device_printf(sc->dev,
 463                     "PCIe x%d Link, expect reduced performance\n",
 464                     sc->link_width);
 465         }
 466 #endif
 467         touch_bars(dev);
 468         pci_enable_busmaster(dev);
 469         /*
 470          * Allocate the registers and make them available to the driver.
 471          * The registers that we care about for NIC mode are in BAR 0
 472          */
 473         sc->regs_rid = PCIR_BAR(0);
 474         if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 475             &sc->regs_rid, RF_ACTIVE)) == NULL) {
 476                 device_printf(dev, "Cannot allocate BAR region 0\n");
 477                 return (ENXIO);
 478         }
 479         sc->udbs_rid = PCIR_BAR(2);
 480         sc->udbs_res = NULL;
 481         if (is_offload(sc) &&
 482             ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 483                    &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
 484                 device_printf(dev, "Cannot allocate BAR region 1\n");
 485                 error = ENXIO;
 486                 goto out;
 487         }
 488
 489         snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
 490             device_get_unit(dev));
 491         ADAPTER_LOCK_INIT(sc, sc->lockbuf);
 492
 493         snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
 494             device_get_unit(dev));
 495         snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
 496             device_get_unit(dev));
 497         snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
 498             device_get_unit(dev));
 499
 500         MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
 501         MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
 502         MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
 503
 504         sc->bt = rman_get_bustag(sc->regs_res);
 505         sc->bh = rman_get_bushandle(sc->regs_res);
 506         sc->mmio_len = rman_get_size(sc->regs_res);
 507
 508         for (i = 0; i < MAX_NPORTS; i++)
 509                 sc->port[i].adapter = sc;
 510
 511         if (t3_prep_adapter(sc, ai, 1) < 0) {
 512                 printf("prep adapter failed\n");
 513                 error = ENODEV;
 514                 goto out;
 515         }
 516         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
 517          * enough messages for the queue sets.  If that fails, try falling
 518          * back to MSI.  If that fails, then try falling back to the legacy
 519          * interrupt pin model.
 520          */
 521 #ifdef MSI_SUPPORTED
 522
 523         sc->msix_regs_rid = 0x20;
 524         if ((msi_allowed >= 2) &&
 525             (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 526             &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
 527
 528                 if (multiq)
 529                         port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
 530                 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
 531
 532                 if (pci_msix_count(dev) == 0 ||
 533                     (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
 534                     sc->msi_count != msi_needed) {
 535                         device_printf(dev, "alloc msix failed - "
 536                                       "msi_count=%d, msi_needed=%d, err=%d; "
 537                                       "will try MSI\n", sc->msi_count,
 538                                       msi_needed, error);
 539                         sc->msi_count = 0;
 540                         port_qsets = 1;
 541                         pci_release_msi(dev);
 542                         bus_release_resource(dev, SYS_RES_MEMORY,
 543                             sc->msix_regs_rid, sc->msix_regs_res);
 544                         sc->msix_regs_res = NULL;
 545                 } else {
 546                         sc->flags |= USING_MSIX;
 547                         sc->cxgb_intr = cxgb_async_intr;
 548                         device_printf(dev,
 549                                       "using MSI-X interrupts (%u vectors)\n",
 550                                       sc->msi_count);
 551                 }
 552         }
 553
 554         if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
 555                 sc->msi_count = 1;
 556                 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
 557                         device_printf(dev, "alloc msi failed - "
 558                                       "err=%d; will try INTx\n", error);
 559                         sc->msi_count = 0;
 560                         port_qsets = 1;
 561                         pci_release_msi(dev);
 562                 } else {
 563                         sc->flags |= USING_MSI;
 564                         sc->cxgb_intr = t3_intr_msi;
 565                         device_printf(dev, "using MSI interrupts\n");
 566                 }
 567         }
 568 #endif
 569         if (sc->msi_count == 0) {
 570                 device_printf(dev, "using line interrupts\n");
 571                 sc->cxgb_intr = t3b_intr;
 572         }
 573
 574         /* Create a private taskqueue thread for handling driver events */
 575 #ifdef TASKQUEUE_CURRENT
 576         sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
 577             taskqueue_thread_enqueue, &sc->tq);
 578 #else
 579         sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
 580             taskqueue_thread_enqueue, &sc->tq);
 581 #endif
 582         if (sc->tq == NULL) {
 583                 device_printf(dev, "failed to allocate controller task queue\n");
 584                 goto out;
 585         }
 586
 587         taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
 588             device_get_nameunit(dev));
 589         TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
 590         TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
 591
 592
 593         /* Create a periodic callout for checking adapter status */
 594         callout_init(&sc->cxgb_tick_ch, TRUE);
 595
 596         if (t3_check_fw_version(sc) < 0 || force_fw_update) {
 597                 /*
 598                  * Warn user that a firmware update will be attempted in init.
 599                  */
 600                 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
 601                     FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 602                 sc->flags &= ~FW_UPTODATE;
 603         } else {
 604                 sc->flags |= FW_UPTODATE;
 605         }
 606
 607         if (t3_check_tpsram_version(sc) < 0) {
 608                 /*
 609                  * Warn user that a firmware update will be attempted in init.
 610                  */
 611                 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
 612                     t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 613                 sc->flags &= ~TPS_UPTODATE;
 614         } else {
 615                 sc->flags |= TPS_UPTODATE;
 616         }
 617
 618         /*
 619          * Create a child device for each MAC.  The ethernet attachment
 620          * will be done in these children.
 621          */
 622         for (i = 0; i < (sc)->params.nports; i++) {
 623                 struct port_info *pi;
 624
 625                 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
 626                         device_printf(dev, "failed to add child port\n");
 627                         error = EINVAL;
 628                         goto out;
 629                 }
 630                 pi = &sc->port[i];
 631                 pi->adapter = sc;
 632                 pi->nqsets = port_qsets;
 633                 pi->first_qset = i*port_qsets;
 634                 pi->port_id = i;
 635                 pi->tx_chan = i >= ai->nports0;
 636                 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
 637                 sc->rxpkt_map[pi->txpkt_intf] = i;
 638                 sc->port[i].tx_chan = i >= ai->nports0;
 639                 sc->portdev[i] = child;
 640                 device_set_softc(child, pi);
 641         }
 642         if ((error = bus_generic_attach(dev)) != 0)
 643                 goto out;
 644
 645         /* initialize sge private state */
 646         t3_sge_init_adapter(sc);
 647
 648         t3_led_ready(sc);
 649
 650         cxgb_offload_init();
 651         if (is_offload(sc)) {
 652                 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
 653                 cxgb_adapter_ofld(sc);
 654         }
 655         error = t3_get_fw_version(sc, &vers);
 656         if (error)
 657                 goto out;
 658
 659         snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
 660             G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
 661             G_FW_VERSION_MICRO(vers));
 662
 663         snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
 664                  ai->desc, is_offload(sc) ? "R" : "",
 665                  sc->params.vpd.ec, sc->params.vpd.sn);
 666         device_set_desc_copy(dev, buf);
 667
 668         snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
 669                  sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
 670                  sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
 671
 672         device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
 673         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
 674         t3_add_attach_sysctls(sc);
 675 out:
 676         if (error)
 677                 cxgb_free(sc);
 678
 679         return (error);
 680 }
 681
 682 /*
 683  * The cxgb_controller_detach routine is called with the device is
 684  * unloaded from the system.
 685  */
 686
 687 static int
 688 cxgb_controller_detach(device_t dev)
 689 {
 690         struct adapter *sc;
 691
 692         sc = device_get_softc(dev);
 693
 694         cxgb_free(sc);
 695
 696         return (0);
 697 }
 698
 699 /*
 700  * The cxgb_free() is called by the cxgb_controller_detach() routine
 701  * to tear down the structures that were built up in
 702  * cxgb_controller_attach(), and should be the final piece of work
 703  * done when fully unloading the driver.
 704  *
 705  *
 706  *  1. Shutting down the threads started by the cxgb_controller_attach()
 707  *     routine.
 708  *  2. Stopping the lower level device and all callouts (cxgb_down_locked()).
 709  *  3. Detaching all of the port devices created during the
 710  *     cxgb_controller_attach() routine.
 711  *  4. Removing the device children created via cxgb_controller_attach().
 712  *  5. Releasing PCI resources associated with the device.
 713  *  6. Turning off the offload support, iff it was turned on.
 714  *  7. Destroying the mutexes created in cxgb_controller_attach().
 715  *
 716  */
 717 static void
 718 cxgb_free(struct adapter *sc)
 719 {
 720         int i;
 721
 722         ADAPTER_LOCK(sc);
 723         sc->flags |= CXGB_SHUTDOWN;
 724         ADAPTER_UNLOCK(sc);
 725
 726         /*
 727          * Make sure all child devices are gone.
 728          */
 729         bus_generic_detach(sc->dev);
 730         for (i = 0; i < (sc)->params.nports; i++) {
 731                 if (sc->portdev[i] &&
 732                     device_delete_child(sc->dev, sc->portdev[i]) != 0)
 733                         device_printf(sc->dev, "failed to delete child port\n");
 734         }
 735
 736         /*
 737          * At this point, it is as if cxgb_port_detach has run on all ports, and
 738          * cxgb_down has run on the adapter.  All interrupts have been silenced,
 739          * all open devices have been closed.
 740          */
 741         KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
 742                                            __func__, sc->open_device_map));
 743         for (i = 0; i < sc->params.nports; i++) {
 744                 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
 745                                                   __func__, i));
 746         }
 747
 748         /*
 749          * Finish off the adapter's callouts.
 750          */
 751         callout_drain(&sc->cxgb_tick_ch);
 752         callout_drain(&sc->sge_timer_ch);
 753
 754         /*
 755          * Release resources grabbed under FULL_INIT_DONE by cxgb_up.  The
 756          * sysctls are cleaned up by the kernel linker.
 757          */
 758         if (sc->flags & FULL_INIT_DONE) {
 759                 t3_free_sge_resources(sc);
 760                 sc->flags &= ~FULL_INIT_DONE;
 761         }
 762
 763         /*
 764          * Release all interrupt resources.
 765          */
 766         cxgb_teardown_interrupts(sc);
 767 #ifdef MSI_SUPPORTED
 768         if (sc->flags & (USING_MSI | USING_MSIX)) {
 769                 device_printf(sc->dev, "releasing msi message(s)\n");
 770                 pci_release_msi(sc->dev);
 771         } else {
 772                 device_printf(sc->dev, "no msi message to release\n");
 773         }
 774
 775         if (sc->msix_regs_res != NULL) {
 776                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
 777                     sc->msix_regs_res);
 778         }
 779 #endif
 780
 781         /*
 782          * Free the adapter's taskqueue.
 783          */
 784         if (sc->tq != NULL) {
 785                 taskqueue_free(sc->tq);
 786                 sc->tq = NULL;
 787         }
 788
 789         if (is_offload(sc)) {
 790                 clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
 791                 cxgb_adapter_unofld(sc);
 792         }
 793
 794 #ifdef notyet
 795         if (sc->flags & CXGB_OFLD_INIT)
 796                 cxgb_offload_deactivate(sc);
 797 #endif
 798         free(sc->filters, M_DEVBUF);
 799         t3_sge_free(sc);
 800
 801         cxgb_offload_exit();
 802
 803         if (sc->udbs_res != NULL)
 804                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
 805                     sc->udbs_res);
 806
 807         if (sc->regs_res != NULL)
 808                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
 809                     sc->regs_res);
 810
 811         MTX_DESTROY(&sc->mdio_lock);
 812         MTX_DESTROY(&sc->sge.reg_lock);
 813         MTX_DESTROY(&sc->elmer_lock);
 814         ADAPTER_LOCK_DEINIT(sc);
 815 }
 816
 817 /**
 818  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 819  *      @sc: the controller softc
 820  *
 821  *      Determines how many sets of SGE queues to use and initializes them.
 822  *      We support multiple queue sets per port if we have MSI-X, otherwise
 823  *      just one queue set per port.
 824  */
 825 static int
 826 setup_sge_qsets(adapter_t *sc)
 827 {
 828         int i, j, err, irq_idx = 0, qset_idx = 0;
 829         u_int ntxq = SGE_TXQ_PER_SET;
 830
 831         if ((err = t3_sge_alloc(sc)) != 0) {
 832                 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
 833                 return (err);
 834         }
 835
 836         if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
 837                 irq_idx = -1;
 838
 839         for (i = 0; i < (sc)->params.nports; i++) {
 840                 struct port_info *pi = &sc->port[i];
 841
 842                 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
 843                         err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
 844                             (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
 845                             &sc->params.sge.qset[qset_idx], ntxq, pi);
 846                         if (err) {
 847                                 t3_free_sge_resources(sc);
 848                                 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n",
 849                                     err);
 850                                 return (err);
 851                         }
 852                 }
 853         }
 854
 855         return (0);
 856 }
 857
 858 static void
 859 cxgb_teardown_interrupts(adapter_t *sc)
 860 {
 861         int i;
 862
 863         for (i = 0; i < SGE_QSETS; i++) {
 864                 if (sc->msix_intr_tag[i] == NULL) {
 865
 866                         /* Should have been setup fully or not at all */
 867                         KASSERT(sc->msix_irq_res[i] == NULL &&
 868                                 sc->msix_irq_rid[i] == 0,
 869                                 ("%s: half-done interrupt (%d).", __func__, i));
 870
 871                         continue;
 872                 }
 873
 874                 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
 875                                   sc->msix_intr_tag[i]);
 876                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
 877                                      sc->msix_irq_res[i]);
 878
 879                 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
 880                 sc->msix_irq_rid[i] = 0;
 881         }
 882
 883         if (sc->intr_tag) {
 884                 KASSERT(sc->irq_res != NULL,
 885                         ("%s: half-done interrupt.", __func__));
 886
 887                 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
 888                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
 889                                      sc->irq_res);
 890
 891                 sc->irq_res = sc->intr_tag = NULL;
 892                 sc->irq_rid = 0;
 893         }
 894 }
 895
 896 static int
 897 cxgb_setup_interrupts(adapter_t *sc)
 898 {
 899         struct resource *res;
 900         void *tag;
 901         int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
 902
 903         sc->irq_rid = intr_flag ? 1 : 0;
 904         sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
 905                                              RF_SHAREABLE | RF_ACTIVE);
 906         if (sc->irq_res == NULL) {
 907                 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
 908                               intr_flag, sc->irq_rid);
 909                 err = EINVAL;
 910                 sc->irq_rid = 0;
 911         } else {
 912                 err = bus_setup_intr(sc->dev, sc->irq_res,
 913                                      INTR_MPSAFE | INTR_TYPE_NET,
 914 #ifdef INTR_FILTERS
 915                                      NULL,
 916 #endif
 917                                      sc->cxgb_intr, sc, &sc->intr_tag);
 918
 919                 if (err) {
 920                         device_printf(sc->dev,
 921                                       "Cannot set up interrupt (%x, %u, %d)\n",
 922                                       intr_flag, sc->irq_rid, err);
 923                         bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
 924                                              sc->irq_res);
 925                         sc->irq_res = sc->intr_tag = NULL;
 926                         sc->irq_rid = 0;
 927                 }
 928         }
 929
 930         /* That's all for INTx or MSI */
 931         if (!(intr_flag & USING_MSIX) || err)
 932                 return (err);
 933
 934         for (i = 0; i < sc->msi_count - 1; i++) {
 935                 rid = i + 2;
 936                 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
 937                                              RF_SHAREABLE | RF_ACTIVE);
 938                 if (res == NULL) {
 939                         device_printf(sc->dev, "Cannot allocate interrupt "
 940                                       "for message %d\n", rid);
 941                         err = EINVAL;
 942                         break;
 943                 }
 944
 945                 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
 946 #ifdef INTR_FILTERS
 947                                      NULL,
 948 #endif
 949                                      t3_intr_msix, &sc->sge.qs[i], &tag);
 950                 if (err) {
 951                         device_printf(sc->dev, "Cannot set up interrupt "
 952                                       "for message %d (%d)\n", rid, err);
 953                         bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
 954                         break;
 955                 }
 956
 957                 sc->msix_irq_rid[i] = rid;
 958                 sc->msix_irq_res[i] = res;
 959                 sc->msix_intr_tag[i] = tag;
 960         }
 961
 962         if (err)
 963                 cxgb_teardown_interrupts(sc);
 964
 965         return (err);
 966 }
 967
 968
 969 static int
 970 cxgb_port_probe(device_t dev)
 971 {
 972         struct port_info *p;
 973         char buf[80];
 974         const char *desc;
 975
 976         p = device_get_softc(dev);
 977         desc = p->phy.desc;
 978         snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
 979         device_set_desc_copy(dev, buf);
 980         return (0);
 981 }
 982
 983
 984 static int
 985 cxgb_makedev(struct port_info *pi)
 986 {
 987
 988         pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
 989             UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
 990
 991         if (pi->port_cdev == NULL)
 992                 return (ENOMEM);
 993
 994         pi->port_cdev->si_drv1 = (void *)pi;
 995
 996         return (0);
 997 }
 998
 999 #ifndef LRO_SUPPORTED
1000 #ifdef IFCAP_LRO
1001 #undef IFCAP_LRO
1002 #endif
1003 #define IFCAP_LRO 0x0
1004 #endif
1005
1006 #ifdef TSO_SUPPORTED
1007 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO)
1008 /* Don't enable TSO6 yet */
1009 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO)
1010 #else
1011 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
1012 /* Don't enable TSO6 yet */
1013 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM |  IFCAP_JUMBO_MTU)
1014 #define IFCAP_TSO4 0x0
1015 #define IFCAP_TSO6 0x0
1016 #define CSUM_TSO   0x0
1017 #endif
1018
1019
1020 static int
1021 cxgb_port_attach(device_t dev)
1022 {
1023         struct port_info *p;
1024         struct ifnet *ifp;
1025         int err;
1026         struct adapter *sc;
1027
1028
1029         p = device_get_softc(dev);
1030         sc = p->adapter;
1031         snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1032             device_get_unit(device_get_parent(dev)), p->port_id);
1033         PORT_LOCK_INIT(p, p->lockbuf);
1034
1035         /* Allocate an ifnet object and set it up */
1036         ifp = p->ifp = if_alloc(IFT_ETHER);
1037         if (ifp == NULL) {
1038                 device_printf(dev, "Cannot allocate ifnet\n");
1039                 return (ENOMEM);
1040         }
1041
1042         /*
1043          * Note that there is currently no watchdog timer.
1044          */
1045         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1046         ifp->if_init = cxgb_init;
1047         ifp->if_softc = p;
1048         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1049         ifp->if_ioctl = cxgb_ioctl;
1050         ifp->if_start = cxgb_start;
1051
1052
1053         ifp->if_timer = 0;      /* Disable ifnet watchdog */
1054         ifp->if_watchdog = NULL;
1055
1056         ifp->if_snd.ifq_drv_maxlen = cxgb_snd_queue_len;
1057         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
1058         IFQ_SET_READY(&ifp->if_snd);
1059
1060         ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
1061         ifp->if_capabilities |= CXGB_CAP;
1062         ifp->if_capenable |= CXGB_CAP_ENABLE;
1063         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
1064         /*
1065          * disable TSO on 4-port - it isn't supported by the firmware yet
1066          */
1067         if (p->adapter->params.nports > 2) {
1068                 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6);
1069                 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6);
1070                 ifp->if_hwassist &= ~CSUM_TSO;
1071         }
1072
1073         ether_ifattach(ifp, p->hw_addr);
1074         ifp->if_transmit = cxgb_transmit;
1075         ifp->if_qflush = cxgb_qflush;
1076
1077         /*
1078          * Only default to jumbo frames on 10GigE
1079          */
1080         if (p->adapter->params.nports <= 2)
1081                 ifp->if_mtu = ETHERMTU_JUMBO;
1082         if ((err = cxgb_makedev(p)) != 0) {
1083                 printf("makedev failed %d\n", err);
1084                 return (err);
1085         }
1086
1087         /* Create a list of media supported by this port */
1088         ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1089             cxgb_media_status);
1090         cxgb_build_medialist(p);
1091
1092         t3_sge_init_port(p);
1093
1094         return (err);
1095 }
1096
1097 /*
1098  * cxgb_port_detach() is called via the device_detach methods when
1099  * cxgb_free() calls the bus_generic_detach.  It is responsible for
1100  * removing the device from the view of the kernel, i.e. from all
1101  * interfaces lists etc.  This routine is only called when the driver is
1102  * being unloaded, not when the link goes down.
1103  */
1104 static int
1105 cxgb_port_detach(device_t dev)
1106 {
1107         struct port_info *p;
1108         struct adapter *sc;
1109         int i;
1110
1111         p = device_get_softc(dev);
1112         sc = p->adapter;
1113
1114         /* Tell cxgb_ioctl and if_init that the port is going away */
1115         ADAPTER_LOCK(sc);
1116         SET_DOOMED(p);
1117         wakeup(&sc->flags);
1118         while (IS_BUSY(sc))
1119                 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1120         SET_BUSY(sc);
1121         ADAPTER_UNLOCK(sc);
1122
1123         if (p->port_cdev != NULL)
1124                 destroy_dev(p->port_cdev);
1125
1126         cxgb_uninit_synchronized(p);
1127         ether_ifdetach(p->ifp);
1128
1129         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1130                 struct sge_qset *qs = &sc->sge.qs[i];
1131                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1132
1133                 callout_drain(&txq->txq_watchdog);
1134                 callout_drain(&txq->txq_timer);
1135         }
1136
1137         PORT_LOCK_DEINIT(p);
1138         if_free(p->ifp);
1139         p->ifp = NULL;
1140
1141         ADAPTER_LOCK(sc);
1142         CLR_BUSY(sc);
1143         wakeup_one(&sc->flags);
1144         ADAPTER_UNLOCK(sc);
1145         return (0);
1146 }
1147
1148 void
1149 t3_fatal_err(struct adapter *sc)
1150 {
1151         u_int fw_status[4];
1152
1153         if (sc->flags & FULL_INIT_DONE) {
1154                 t3_sge_stop(sc);
1155                 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1156                 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1157                 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1158                 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1159                 t3_intr_disable(sc);
1160         }
1161         device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1162         if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1163                 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1164                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1165 }
1166
1167 int
1168 t3_os_find_pci_capability(adapter_t *sc, int cap)
1169 {
1170         device_t dev;
1171         struct pci_devinfo *dinfo;
1172         pcicfgregs *cfg;
1173         uint32_t status;
1174         uint8_t ptr;
1175
1176         dev = sc->dev;
1177         dinfo = device_get_ivars(dev);
1178         cfg = &dinfo->cfg;
1179
1180         status = pci_read_config(dev, PCIR_STATUS, 2);
1181         if (!(status & PCIM_STATUS_CAPPRESENT))
1182                 return (0);
1183
1184         switch (cfg->hdrtype & PCIM_HDRTYPE) {
1185         case 0:
1186         case 1:
1187                 ptr = PCIR_CAP_PTR;
1188                 break;
1189         case 2:
1190                 ptr = PCIR_CAP_PTR_2;
1191                 break;
1192         default:
1193                 return (0);
1194                 break;
1195         }
1196         ptr = pci_read_config(dev, ptr, 1);
1197
1198         while (ptr != 0) {
1199                 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1200                         return (ptr);
1201                 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1202         }
1203
1204         return (0);
1205 }
1206
1207 int
1208 t3_os_pci_save_state(struct adapter *sc)
1209 {
1210         device_t dev;
1211         struct pci_devinfo *dinfo;
1212
1213         dev = sc->dev;
1214         dinfo = device_get_ivars(dev);
1215
1216         pci_cfg_save(dev, dinfo, 0);
1217         return (0);
1218 }
1219
1220 int
1221 t3_os_pci_restore_state(struct adapter *sc)
1222 {
1223         device_t dev;
1224         struct pci_devinfo *dinfo;
1225
1226         dev = sc->dev;
1227         dinfo = device_get_ivars(dev);
1228
1229         pci_cfg_restore(dev, dinfo);
1230         return (0);
1231 }
1232
1233 /**
1234  *      t3_os_link_changed - handle link status changes
1235  *      @sc: the adapter associated with the link change
1236  *      @port_id: the port index whose link status has changed
1237  *      @link_status: the new status of the link
1238  *      @speed: the new speed setting
1239  *      @duplex: the new duplex setting
1240  *      @fc: the new flow-control setting
1241  *
1242  *      This is the OS-dependent handler for link status changes.  The OS
1243  *      neutral handler takes care of most of the processing for these events,
1244  *      then calls this handler for any OS-specific processing.
1245  */
1246 void
1247 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1248      int duplex, int fc, int mac_was_reset)
1249 {
1250         struct port_info *pi = &adapter->port[port_id];
1251         struct ifnet *ifp = pi->ifp;
1252
1253         /* no race with detach, so ifp should always be good */
1254         KASSERT(ifp, ("%s: if detached.", __func__));
1255
1256         /* Reapply mac settings if they were lost due to a reset */
1257         if (mac_was_reset) {
1258                 PORT_LOCK(pi);
1259                 cxgb_update_mac_settings(pi);
1260                 PORT_UNLOCK(pi);
1261         }
1262
1263         if (link_status) {
1264                 ifp->if_baudrate = IF_Mbps(speed);
1265                 if_link_state_change(ifp, LINK_STATE_UP);
1266         } else
1267                 if_link_state_change(ifp, LINK_STATE_DOWN);
1268 }
1269
1270 /**
1271  *      t3_os_phymod_changed - handle PHY module changes
1272  *      @phy: the PHY reporting the module change
1273  *      @mod_type: new module type
1274  *
1275  *      This is the OS-dependent handler for PHY module changes.  It is
1276  *      invoked when a PHY module is removed or inserted for any OS-specific
1277  *      processing.
1278  */
1279 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1280 {
1281         static const char *mod_str[] = {
1282                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
1283         };
1284         struct port_info *pi = &adap->port[port_id];
1285         int mod = pi->phy.modtype;
1286
1287         if (mod != pi->media.ifm_cur->ifm_data)
1288                 cxgb_build_medialist(pi);
1289
1290         if (mod == phy_modtype_none)
1291                 if_printf(pi->ifp, "PHY module unplugged\n");
1292         else {
1293                 KASSERT(mod < ARRAY_SIZE(mod_str),
1294                         ("invalid PHY module type %d", mod));
1295                 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1296         }
1297 }
1298
1299 /*
1300  * Interrupt-context handler for external (PHY) interrupts.
1301  */
1302 void
1303 t3_os_ext_intr_handler(adapter_t *sc)
1304 {
1305         if (cxgb_debug)
1306                 printf("t3_os_ext_intr_handler\n");
1307         /*
1308          * Schedule a task to handle external interrupts as they may be slow
1309          * and we use a mutex to protect MDIO registers.  We disable PHY
1310          * interrupts in the meantime and let the task reenable them when
1311          * it's done.
1312          */
1313         if (sc->slow_intr_mask) {
1314                 ADAPTER_LOCK(sc);
1315                 sc->slow_intr_mask &= ~F_T3DBG;
1316                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1317                 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1318                 ADAPTER_UNLOCK(sc);
1319         }
1320 }
1321
1322 void
1323 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1324 {
1325
1326         /*
1327          * The ifnet might not be allocated before this gets called,
1328          * as this is called early on in attach by t3_prep_adapter
1329          * save the address off in the port structure
1330          */
1331         if (cxgb_debug)
1332                 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1333         bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1334 }
1335
1336 /*
1337  * Programs the XGMAC based on the settings in the ifnet.  These settings
1338  * include MTU, MAC address, mcast addresses, etc.
1339  */
1340 static void
1341 cxgb_update_mac_settings(struct port_info *p)
1342 {
1343         struct ifnet *ifp = p->ifp;
1344         struct t3_rx_mode rm;
1345         struct cmac *mac = &p->mac;
1346         int mtu, hwtagging;
1347
1348         PORT_LOCK_ASSERT_OWNED(p);
1349
1350         bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1351
1352         mtu = ifp->if_mtu;
1353         if (ifp->if_capenable & IFCAP_VLAN_MTU)
1354                 mtu += ETHER_VLAN_ENCAP_LEN;
1355
1356         hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1357
1358         t3_mac_set_mtu(mac, mtu);
1359         t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1360         t3_mac_set_address(mac, 0, p->hw_addr);
1361         t3_init_rx_mode(&rm, p);
1362         t3_mac_set_rx_mode(mac, &rm);
1363 }
1364
1365
1366 static int
1367 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1368                               unsigned long n)
1369 {
1370         int attempts = 5;
1371
1372         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1373                 if (!--attempts)
1374                         return (ETIMEDOUT);
1375                 t3_os_sleep(10);
1376         }
1377         return 0;
1378 }
1379
1380 static int
1381 init_tp_parity(struct adapter *adap)
1382 {
1383         int i;
1384         struct mbuf *m;
1385         struct cpl_set_tcb_field *greq;
1386         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1387
1388         t3_tp_set_offload_mode(adap, 1);
1389
1390         for (i = 0; i < 16; i++) {
1391                 struct cpl_smt_write_req *req;
1392
1393                 m = m_gethdr(M_WAITOK, MT_DATA);
1394                 req = mtod(m, struct cpl_smt_write_req *);
1395                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1396                 memset(req, 0, sizeof(*req));
1397                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1398                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1399                 req->iff = i;
1400                 t3_mgmt_tx(adap, m);
1401         }
1402
1403         for (i = 0; i < 2048; i++) {
1404                 struct cpl_l2t_write_req *req;
1405
1406                 m = m_gethdr(M_WAITOK, MT_DATA);
1407                 req = mtod(m, struct cpl_l2t_write_req *);
1408                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1409                 memset(req, 0, sizeof(*req));
1410                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1411                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1412                 req->params = htonl(V_L2T_W_IDX(i));
1413                 t3_mgmt_tx(adap, m);
1414         }
1415
1416         for (i = 0; i < 2048; i++) {
1417                 struct cpl_rte_write_req *req;
1418
1419                 m = m_gethdr(M_WAITOK, MT_DATA);
1420                 req = mtod(m, struct cpl_rte_write_req *);
1421                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1422                 memset(req, 0, sizeof(*req));
1423                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1424                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1425                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1426                 t3_mgmt_tx(adap, m);
1427         }
1428
1429         m = m_gethdr(M_WAITOK, MT_DATA);
1430         greq = mtod(m, struct cpl_set_tcb_field *);
1431         m->m_len = m->m_pkthdr.len = sizeof(*greq);
1432         memset(greq, 0, sizeof(*greq));
1433         greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1434         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1435         greq->mask = htobe64(1);
1436         t3_mgmt_tx(adap, m);
1437
1438         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1439         t3_tp_set_offload_mode(adap, 0);
1440         return (i);
1441 }
1442
1443 /**
1444  *      setup_rss - configure Receive Side Steering (per-queue connection demux)
1445  *      @adap: the adapter
1446  *
1447  *      Sets up RSS to distribute packets to multiple receive queues.  We
1448  *      configure the RSS CPU lookup table to distribute to the number of HW
1449  *      receive queues, and the response queue lookup table to narrow that
1450  *      down to the response queues actually configured for each port.
1451  *      We always configure the RSS mapping for two ports since the mapping
1452  *      table has plenty of entries.
1453  */
1454 static void
1455 setup_rss(adapter_t *adap)
1456 {
1457         int i;
1458         u_int nq[2];
1459         uint8_t cpus[SGE_QSETS + 1];
1460         uint16_t rspq_map[RSS_TABLE_SIZE];
1461
1462         for (i = 0; i < SGE_QSETS; ++i)
1463                 cpus[i] = i;
1464         cpus[SGE_QSETS] = 0xff;
1465
1466         nq[0] = nq[1] = 0;
1467         for_each_port(adap, i) {
1468                 const struct port_info *pi = adap2pinfo(adap, i);
1469
1470                 nq[pi->tx_chan] += pi->nqsets;
1471         }
1472         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1473                 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1474                 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1475         }
1476
1477         /* Calculate the reverse RSS map table */
1478         for (i = 0; i < SGE_QSETS; ++i)
1479                 adap->rrss_map[i] = 0xff;
1480         for (i = 0; i < RSS_TABLE_SIZE; ++i)
1481                 if (adap->rrss_map[rspq_map[i]] == 0xff)
1482                         adap->rrss_map[rspq_map[i]] = i;
1483
1484         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1485                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1486                       F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1487                       cpus, rspq_map);
1488
1489 }
1490
1491 /*
1492  * Sends an mbuf to an offload queue driver
1493  * after dealing with any active network taps.
1494  */
1495 static inline int
1496 offload_tx(struct t3cdev *tdev, struct mbuf *m)
1497 {
1498         int ret;
1499
1500         ret = t3_offload_tx(tdev, m);
1501         return (ret);
1502 }
1503
1504 static int
1505 write_smt_entry(struct adapter *adapter, int idx)
1506 {
1507         struct port_info *pi = &adapter->port[idx];
1508         struct cpl_smt_write_req *req;
1509         struct mbuf *m;
1510
1511         if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1512                 return (ENOMEM);
1513
1514         req = mtod(m, struct cpl_smt_write_req *);
1515         m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
1516
1517         req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1518         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1519         req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
1520         req->iff = idx;
1521         memset(req->src_mac1, 0, sizeof(req->src_mac1));
1522         memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1523
1524         m_set_priority(m, 1);
1525
1526         offload_tx(&adapter->tdev, m);
1527
1528         return (0);
1529 }
1530
1531 static int
1532 init_smt(struct adapter *adapter)
1533 {
1534         int i;
1535
1536         for_each_port(adapter, i)
1537                 write_smt_entry(adapter, i);
1538         return 0;
1539 }
1540
1541 static void
1542 init_port_mtus(adapter_t *adapter)
1543 {
1544         unsigned int mtus = ETHERMTU | (ETHERMTU << 16);
1545
1546         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1547 }
1548
1549 static void
1550 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1551                               int hi, int port)
1552 {
1553         struct mbuf *m;
1554         struct mngt_pktsched_wr *req;
1555
1556         m = m_gethdr(M_DONTWAIT, MT_DATA);
1557         if (m) {
1558                 req = mtod(m, struct mngt_pktsched_wr *);
1559                 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1560                 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1561                 req->sched = sched;
1562                 req->idx = qidx;
1563                 req->min = lo;
1564                 req->max = hi;
1565                 req->binding = port;
1566                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1567                 t3_mgmt_tx(adap, m);
1568         }
1569 }
1570
1571 static void
1572 bind_qsets(adapter_t *sc)
1573 {
1574         int i, j;
1575
1576         for (i = 0; i < (sc)->params.nports; ++i) {
1577                 const struct port_info *pi = adap2pinfo(sc, i);
1578
1579                 for (j = 0; j < pi->nqsets; ++j) {
1580                         send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1581                                           -1, pi->tx_chan);
1582
1583                 }
1584         }
1585 }
1586
1587 static void
1588 update_tpeeprom(struct adapter *adap)
1589 {
1590 #ifdef FIRMWARE_LATEST
1591         const struct firmware *tpeeprom;
1592 #else
1593         struct firmware *tpeeprom;
1594 #endif
1595
1596         uint32_t version;
1597         unsigned int major, minor;
1598         int ret, len;
1599         char rev, name[32];
1600
1601         t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1602
1603         major = G_TP_VERSION_MAJOR(version);
1604         minor = G_TP_VERSION_MINOR(version);
1605         if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
1606                 return;
1607
1608         rev = t3rev2char(adap);
1609         snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1610
1611         tpeeprom = firmware_get(name);
1612         if (tpeeprom == NULL) {
1613                 device_printf(adap->dev,
1614                               "could not load TP EEPROM: unable to load %s\n",
1615                               name);
1616                 return;
1617         }
1618
1619         len = tpeeprom->datasize - 4;
1620
1621         ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1622         if (ret)
1623                 goto release_tpeeprom;
1624
1625         if (len != TP_SRAM_LEN) {
1626                 device_printf(adap->dev,
1627                               "%s length is wrong len=%d expected=%d\n", name,
1628                               len, TP_SRAM_LEN);
1629                 return;
1630         }
1631
1632         ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1633             TP_SRAM_OFFSET);
1634
1635         if (!ret) {
1636                 device_printf(adap->dev,
1637                         "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1638                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1639         } else
1640                 device_printf(adap->dev,
1641                               "Protocol SRAM image update in EEPROM failed\n");
1642
1643 release_tpeeprom:
1644         firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1645
1646         return;
1647 }
1648
1649 static int
1650 update_tpsram(struct adapter *adap)
1651 {
1652 #ifdef FIRMWARE_LATEST
1653         const struct firmware *tpsram;
1654 #else
1655         struct firmware *tpsram;
1656 #endif
1657         int ret;
1658         char rev, name[32];
1659
1660         rev = t3rev2char(adap);
1661         snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1662
1663         update_tpeeprom(adap);
1664
1665         tpsram = firmware_get(name);
1666         if (tpsram == NULL){
1667                 device_printf(adap->dev, "could not load TP SRAM\n");
1668                 return (EINVAL);
1669         } else
1670                 device_printf(adap->dev, "updating TP SRAM\n");
1671
1672         ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1673         if (ret)
1674                 goto release_tpsram;
1675
1676         ret = t3_set_proto_sram(adap, tpsram->data);
1677         if (ret)
1678                 device_printf(adap->dev, "loading protocol SRAM failed\n");
1679
1680 release_tpsram:
1681         firmware_put(tpsram, FIRMWARE_UNLOAD);
1682
1683         return ret;
1684 }
1685
1686 /**
1687  *      cxgb_up - enable the adapter
1688  *      @adap: adapter being enabled
1689  *
1690  *      Called when the first port is enabled, this function performs the
1691  *      actions necessary to make an adapter operational, such as completing
1692  *      the initialization of HW modules, and enabling interrupts.
1693  */
1694 static int
1695 cxgb_up(struct adapter *sc)
1696 {
1697         int err = 0;
1698
1699         KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1700                                            __func__, sc->open_device_map));
1701
1702         if ((sc->flags & FULL_INIT_DONE) == 0) {
1703
1704                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1705
1706                 if ((sc->flags & FW_UPTODATE) == 0)
1707                         if ((err = upgrade_fw(sc)))
1708                                 goto out;
1709
1710                 if ((sc->flags & TPS_UPTODATE) == 0)
1711                         if ((err = update_tpsram(sc)))
1712                                 goto out;
1713
1714                 err = t3_init_hw(sc, 0);
1715                 if (err)
1716                         goto out;
1717
1718                 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1719                 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1720
1721                 err = setup_sge_qsets(sc);
1722                 if (err)
1723                         goto out;
1724
1725                 setup_rss(sc);
1726
1727                 t3_intr_clear(sc);
1728                 err = cxgb_setup_interrupts(sc);
1729                 if (err)
1730                         goto out;
1731
1732                 t3_add_configured_sysctls(sc);
1733                 sc->flags |= FULL_INIT_DONE;
1734         }
1735
1736         t3_intr_clear(sc);
1737         t3_sge_start(sc);
1738         t3_intr_enable(sc);
1739
1740         if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1741             is_offload(sc) && init_tp_parity(sc) == 0)
1742                 sc->flags |= TP_PARITY_INIT;
1743
1744         if (sc->flags & TP_PARITY_INIT) {
1745                 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1746                 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1747         }
1748
1749         if (!(sc->flags & QUEUES_BOUND)) {
1750                 bind_qsets(sc);
1751                 sc->flags |= QUEUES_BOUND;
1752         }
1753
1754         t3_sge_reset_adapter(sc);
1755 out:
1756         return (err);
1757 }
1758
1759 /*
1760  * Called when the last open device is closed.  Does NOT undo all of cxgb_up's
1761  * work.  Specifically, the resources grabbed under FULL_INIT_DONE are released
1762  * during controller_detach, not here.
1763  */
1764 static void
1765 cxgb_down(struct adapter *sc)
1766 {
1767         t3_sge_stop(sc);
1768         t3_intr_disable(sc);
1769 }
1770
1771 static int
1772 offload_open(struct port_info *pi)
1773 {
1774         struct adapter *sc = pi->adapter;
1775         struct t3cdev *tdev = &sc->tdev;
1776
1777         setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT);
1778
1779         t3_tp_set_offload_mode(sc, 1);
1780         tdev->lldev = pi->ifp;
1781         init_port_mtus(sc);
1782         t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
1783                      sc->params.rev == 0 ?  sc->port[0].ifp->if_mtu : 0xffff);
1784         init_smt(sc);
1785         cxgb_add_clients(tdev);
1786
1787         return (0);
1788 }
1789
1790 static int
1791 offload_close(struct t3cdev *tdev)
1792 {
1793         struct adapter *adapter = tdev2adap(tdev);
1794
1795         if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1796                 return (0);
1797
1798         /* Call back all registered clients */
1799         cxgb_remove_clients(tdev);
1800
1801         tdev->lldev = NULL;
1802         cxgb_set_dummy_ops(tdev);
1803         t3_tp_set_offload_mode(adapter, 0);
1804
1805         clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1806
1807         return (0);
1808 }
1809
1810 /*
1811  * if_init for cxgb ports.
1812  */
1813 static void
1814 cxgb_init(void *arg)
1815 {
1816         struct port_info *p = arg;
1817         struct adapter *sc = p->adapter;
1818
1819         ADAPTER_LOCK(sc);
1820         cxgb_init_locked(p); /* releases adapter lock */
1821         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1822 }
1823
1824 static int
1825 cxgb_init_locked(struct port_info *p)
1826 {
1827         struct adapter *sc = p->adapter;
1828         struct ifnet *ifp = p->ifp;
1829         struct cmac *mac = &p->mac;
1830         int i, rc = 0, may_sleep = 0;
1831
1832         ADAPTER_LOCK_ASSERT_OWNED(sc);
1833
1834         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1835                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1836                         rc = EINTR;
1837                         goto done;
1838                 }
1839         }
1840         if (IS_DOOMED(p)) {
1841                 rc = ENXIO;
1842                 goto done;
1843         }
1844         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1845
1846         /*
1847          * The code that runs during one-time adapter initialization can sleep
1848          * so it's important not to hold any locks across it.
1849          */
1850         may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1851
1852         if (may_sleep) {
1853                 SET_BUSY(sc);
1854                 ADAPTER_UNLOCK(sc);
1855         }
1856
1857         if (sc->open_device_map == 0) {
1858                 if ((rc = cxgb_up(sc)) != 0)
1859                         goto done;
1860
1861                 if (is_offload(sc) && !ofld_disable && offload_open(p))
1862                         log(LOG_WARNING,
1863                             "Could not initialize offload capabilities\n");
1864         }
1865
1866         PORT_LOCK(p);
1867         if (isset(&sc->open_device_map, p->port_id) &&
1868             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1869                 PORT_UNLOCK(p);
1870                 goto done;
1871         }
1872         t3_port_intr_enable(sc, p->port_id);
1873         if (!mac->multiport)
1874                 t3_mac_init(mac);
1875         cxgb_update_mac_settings(p);
1876         t3_link_start(&p->phy, mac, &p->link_config);
1877         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1878         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1879         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1880         PORT_UNLOCK(p);
1881
1882         t3_link_changed(sc, p->port_id);
1883
1884         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1885                 struct sge_qset *qs = &sc->sge.qs[i];
1886                 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1887
1888                 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1889                                  txq->txq_watchdog.c_cpu);
1890         }
1891
1892         /* all ok */
1893         setbit(&sc->open_device_map, p->port_id);
1894
1895 done:
1896         if (may_sleep) {
1897                 ADAPTER_LOCK(sc);
1898                 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1899                 CLR_BUSY(sc);
1900                 wakeup_one(&sc->flags);
1901         }
1902         ADAPTER_UNLOCK(sc);
1903         return (rc);
1904 }
1905
1906 static int
1907 cxgb_uninit_locked(struct port_info *p)
1908 {
1909         struct adapter *sc = p->adapter;
1910         int rc;
1911
1912         ADAPTER_LOCK_ASSERT_OWNED(sc);
1913
1914         while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1915                 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1916                         rc = EINTR;
1917                         goto done;
1918                 }
1919         }
1920         if (IS_DOOMED(p)) {
1921                 rc = ENXIO;
1922                 goto done;
1923         }
1924         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1925         SET_BUSY(sc);
1926         ADAPTER_UNLOCK(sc);
1927
1928         rc = cxgb_uninit_synchronized(p);
1929
1930         ADAPTER_LOCK(sc);
1931         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1932         CLR_BUSY(sc);
1933         wakeup_one(&sc->flags);
1934 done:
1935         ADAPTER_UNLOCK(sc);
1936         return (rc);
1937 }
1938
1939 /*
1940  * Called on "ifconfig down", and from port_detach
1941  */
1942 static int
1943 cxgb_uninit_synchronized(struct port_info *pi)
1944 {
1945         struct adapter *sc = pi->adapter;
1946         struct ifnet *ifp = pi->ifp;
1947
1948         /*
1949          * taskqueue_drain may cause a deadlock if the adapter lock is held.
1950          */
1951         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1952
1953         /*
1954          * Clear this port's bit from the open device map, and then drain all
1955          * the tasks that can access/manipulate this port's port_info or ifp.
1956          * We disable this port's interrupts here and so the the slow/ext
1957          * interrupt tasks won't be enqueued.  The tick task will continue to
1958          * be enqueued every second but the runs after this drain will not see
1959          * this port in the open device map.
1960          *
1961          * A well behaved task must take open_device_map into account and ignore
1962          * ports that are not open.
1963          */
1964         clrbit(&sc->open_device_map, pi->port_id);
1965         t3_port_intr_disable(sc, pi->port_id);
1966         taskqueue_drain(sc->tq, &sc->slow_intr_task);
1967         taskqueue_drain(sc->tq, &sc->ext_intr_task);
1968         taskqueue_drain(sc->tq, &sc->tick_task);
1969
1970         PORT_LOCK(pi);
1971         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1972
1973         /* disable pause frames */
1974         t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1975
1976         /* Reset RX FIFO HWM */
1977         t3_set_reg_field(sc, A_XGM_RXFIFO_CFG +  pi->mac.offset,
1978                          V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1979
1980         DELAY(100 * 1000);
1981
1982         /* Wait for TXFIFO empty */
1983         t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1984                         F_TXFIFO_EMPTY, 1, 20, 5);
1985
1986         DELAY(100 * 1000);
1987         t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1988
1989
1990         pi->phy.ops->power_down(&pi->phy, 1);
1991
1992         PORT_UNLOCK(pi);
1993
1994         pi->link_config.link_ok = 0;
1995         t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1996
1997         if ((sc->open_device_map & PORT_MASK) == 0)
1998                 offload_close(&sc->tdev);
1999
2000         if (sc->open_device_map == 0)
2001                 cxgb_down(pi->adapter);
2002
2003         return (0);
2004 }
2005
2006 #ifdef LRO_SUPPORTED
2007 /*
2008  * Mark lro enabled or disabled in all qsets for this port
2009  */
2010 static int
2011 cxgb_set_lro(struct port_info *p, int enabled)
2012 {
2013         int i;
2014         struct adapter *adp = p->adapter;
2015         struct sge_qset *q;
2016
2017         PORT_LOCK_ASSERT_OWNED(p);
2018         for (i = 0; i < p->nqsets; i++) {
2019                 q = &adp->sge.qs[p->first_qset + i];
2020                 q->lro.enabled = (enabled != 0);
2021         }
2022         return (0);
2023 }
2024 #endif
2025
2026 static int
2027 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
2028 {
2029         struct port_info *p = ifp->if_softc;
2030         struct adapter *sc = p->adapter;
2031         struct ifreq *ifr = (struct ifreq *)data;
2032         int flags, error = 0, mtu;
2033         uint32_t mask;
2034
2035         switch (command) {
2036         case SIOCSIFMTU:
2037                 ADAPTER_LOCK(sc);
2038                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
2039                 if (error) {
2040 fail:
2041                         ADAPTER_UNLOCK(sc);
2042                         return (error);
2043                 }
2044
2045                 mtu = ifr->ifr_mtu;
2046                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
2047                         error = EINVAL;
2048                 } else {
2049                         ifp->if_mtu = mtu;
2050                         PORT_LOCK(p);
2051                         cxgb_update_mac_settings(p);
2052                         PORT_UNLOCK(p);
2053                 }
2054                 ADAPTER_UNLOCK(sc);
2055                 break;
2056         case SIOCSIFFLAGS:
2057                 ADAPTER_LOCK(sc);
2058                 if (IS_DOOMED(p)) {
2059                         error = ENXIO;
2060                         goto fail;
2061                 }
2062                 if (ifp->if_flags & IFF_UP) {
2063                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2064                                 flags = p->if_flags;
2065                                 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
2066                                     ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
2067                                         if (IS_BUSY(sc)) {
2068                                                 error = EBUSY;
2069                                                 goto fail;
2070                                         }
2071                                         PORT_LOCK(p);
2072                                         cxgb_update_mac_settings(p);
2073                                         PORT_UNLOCK(p);
2074                                 }
2075                                 ADAPTER_UNLOCK(sc);
2076                         } else
2077                                 error = cxgb_init_locked(p);
2078                         p->if_flags = ifp->if_flags;
2079                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2080                         error = cxgb_uninit_locked(p);
2081
2082                 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2083                 break;
2084         case SIOCADDMULTI:
2085         case SIOCDELMULTI:
2086                 ADAPTER_LOCK(sc);
2087                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
2088                 if (error)
2089                         goto fail;
2090
2091                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2092                         PORT_LOCK(p);
2093                         cxgb_update_mac_settings(p);
2094                         PORT_UNLOCK(p);
2095                 }
2096                 ADAPTER_UNLOCK(sc);
2097
2098                 break;
2099         case SIOCSIFCAP:
2100                 ADAPTER_LOCK(sc);
2101                 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
2102                 if (error)
2103                         goto fail;
2104
2105                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2106                 if (mask & IFCAP_TXCSUM) {
2107                         if (IFCAP_TXCSUM & ifp->if_capenable) {
2108                                 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
2109                                 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
2110                                     | CSUM_IP | CSUM_TSO);
2111                         } else {
2112                                 ifp->if_capenable |= IFCAP_TXCSUM;
2113                                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP
2114                                     | CSUM_IP);
2115                         }
2116                 }
2117                 if (mask & IFCAP_RXCSUM) {
2118                         ifp->if_capenable ^= IFCAP_RXCSUM;
2119                 }
2120                 if (mask & IFCAP_TSO4) {
2121                         if (IFCAP_TSO4 & ifp->if_capenable) {
2122                                 ifp->if_capenable &= ~IFCAP_TSO4;
2123                                 ifp->if_hwassist &= ~CSUM_TSO;
2124                         } else if (IFCAP_TXCSUM & ifp->if_capenable) {
2125                                 ifp->if_capenable |= IFCAP_TSO4;
2126                                 ifp->if_hwassist |= CSUM_TSO;
2127                         } else
2128                                 error = EINVAL;
2129                 }
2130 #ifdef LRO_SUPPORTED
2131                 if (mask & IFCAP_LRO) {
2132                         ifp->if_capenable ^= IFCAP_LRO;
2133
2134                         /* Safe to do this even if cxgb_up not called yet */
2135                         cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2136                 }
2137 #endif
2138                 if (mask & IFCAP_VLAN_HWTAGGING) {
2139                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2140                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2141                                 PORT_LOCK(p);
2142                                 cxgb_update_mac_settings(p);
2143                                 PORT_UNLOCK(p);
2144                         }
2145                 }
2146                 if (mask & IFCAP_VLAN_MTU) {
2147                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
2148                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2149                                 PORT_LOCK(p);
2150                                 cxgb_update_mac_settings(p);
2151                                 PORT_UNLOCK(p);
2152                         }
2153                 }
2154                 if (mask & IFCAP_VLAN_HWCSUM)
2155                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2156
2157 #ifdef VLAN_CAPABILITIES
2158                 VLAN_CAPABILITIES(ifp);
2159 #endif
2160                 ADAPTER_UNLOCK(sc);
2161                 break;
2162         case SIOCSIFMEDIA:
2163         case SIOCGIFMEDIA:
2164                 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2165                 break;
2166         default:
2167                 error = ether_ioctl(ifp, command, data);
2168         }
2169
2170         return (error);
2171 }
2172
2173 static int
2174 cxgb_media_change(struct ifnet *ifp)
2175 {
2176         return (EOPNOTSUPP);
2177 }
2178
2179 /*
2180  * Translates phy->modtype to the correct Ethernet media subtype.
2181  */
2182 static int
2183 cxgb_ifm_type(int mod)
2184 {
2185         switch (mod) {
2186         case phy_modtype_sr:
2187                 return (IFM_10G_SR);
2188         case phy_modtype_lr:
2189                 return (IFM_10G_LR);
2190         case phy_modtype_lrm:
2191                 return (IFM_10G_LRM);
2192         case phy_modtype_twinax:
2193                 return (IFM_10G_TWINAX);
2194         case phy_modtype_twinax_long:
2195                 return (IFM_10G_TWINAX_LONG);
2196         case phy_modtype_none:
2197                 return (IFM_NONE);
2198         case phy_modtype_unknown:
2199                 return (IFM_UNKNOWN);
2200         }
2201
2202         KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2203         return (IFM_UNKNOWN);
2204 }
2205
2206 /*
2207  * Rebuilds the ifmedia list for this port, and sets the current media.
2208  */
2209 static void
2210 cxgb_build_medialist(struct port_info *p)
2211 {
2212         struct cphy *phy = &p->phy;
2213         struct ifmedia *media = &p->media;
2214         int mod = phy->modtype;
2215         int m = IFM_ETHER | IFM_FDX;
2216
2217         PORT_LOCK(p);
2218
2219         ifmedia_removeall(media);
2220         if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2221                 /* Copper (RJ45) */
2222
2223                 if (phy->caps & SUPPORTED_10000baseT_Full)
2224                         ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2225
2226                 if (phy->caps & SUPPORTED_1000baseT_Full)
2227                         ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2228
2229                 if (phy->caps & SUPPORTED_100baseT_Full)
2230                         ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2231
2232                 if (phy->caps & SUPPORTED_10baseT_Full)
2233                         ifmedia_add(media, m | IFM_10_T, mod, NULL);
2234
2235                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2236                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2237
2238         } else if (phy->caps & SUPPORTED_TP) {
2239                 /* Copper (CX4) */
2240
2241                 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2242                         ("%s: unexpected cap 0x%x", __func__, phy->caps));
2243
2244                 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2245                 ifmedia_set(media, m | IFM_10G_CX4);
2246
2247         } else if (phy->caps & SUPPORTED_FIBRE &&
2248                    phy->caps & SUPPORTED_10000baseT_Full) {
2249                 /* 10G optical (but includes SFP+ twinax) */
2250
2251                 m |= cxgb_ifm_type(mod);
2252                 if (IFM_SUBTYPE(m) == IFM_NONE)
2253                         m &= ~IFM_FDX;
2254
2255                 ifmedia_add(media, m, mod, NULL);
2256                 ifmedia_set(media, m);
2257
2258         } else if (phy->caps & SUPPORTED_FIBRE &&
2259                    phy->caps & SUPPORTED_1000baseT_Full) {
2260                 /* 1G optical */
2261
2262                 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2263                 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2264                 ifmedia_set(media, m | IFM_1000_SX);
2265
2266         } else {
2267                 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2268                             phy->caps));
2269         }
2270
2271         PORT_UNLOCK(p);
2272 }
2273
2274 static void
2275 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2276 {
2277         struct port_info *p = ifp->if_softc;
2278         struct ifmedia_entry *cur = p->media.ifm_cur;
2279         int speed = p->link_config.speed;
2280
2281         if (cur->ifm_data != p->phy.modtype) {
2282                 cxgb_build_medialist(p);
2283                 cur = p->media.ifm_cur;
2284         }
2285
2286         ifmr->ifm_status = IFM_AVALID;
2287         if (!p->link_config.link_ok)
2288                 return;
2289
2290         ifmr->ifm_status |= IFM_ACTIVE;
2291
2292         /*
2293          * active and current will differ iff current media is autoselect.  That
2294          * can happen only for copper RJ45.
2295          */
2296         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2297                 return;
2298         KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2299                 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2300
2301         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2302         if (speed == SPEED_10000)
2303                 ifmr->ifm_active |= IFM_10G_T;
2304         else if (speed == SPEED_1000)
2305                 ifmr->ifm_active |= IFM_1000_T;
2306         else if (speed == SPEED_100)
2307                 ifmr->ifm_active |= IFM_100_TX;
2308         else if (speed == SPEED_10)
2309                 ifmr->ifm_active |= IFM_10_T;
2310         else
2311                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2312                             speed));
2313 }
2314
2315 static void
2316 cxgb_async_intr(void *data)
2317 {
2318         adapter_t *sc = data;
2319
2320         if (cxgb_debug)
2321                 device_printf(sc->dev, "cxgb_async_intr\n");
2322         /*
2323          * May need to sleep - defer to taskqueue
2324          */
2325         taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2326 }
2327
2328 static void
2329 cxgb_ext_intr_handler(void *arg, int count)
2330 {
2331         adapter_t *sc = (adapter_t *)arg;
2332
2333         if (cxgb_debug)
2334                 printf("cxgb_ext_intr_handler\n");
2335
2336         t3_phy_intr_handler(sc);
2337
2338         /* Now reenable external interrupts */
2339         ADAPTER_LOCK(sc);
2340         if (sc->slow_intr_mask) {
2341                 sc->slow_intr_mask |= F_T3DBG;
2342                 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
2343                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
2344         }
2345         ADAPTER_UNLOCK(sc);
2346 }
2347
2348 static inline int
2349 link_poll_needed(struct port_info *p)
2350 {
2351         struct cphy *phy = &p->phy;
2352
2353         if (phy->caps & POLL_LINK_1ST_TIME) {
2354                 p->phy.caps &= ~POLL_LINK_1ST_TIME;
2355                 return (1);
2356         }
2357
2358         return (p->link_fault || !(phy->caps & SUPPORTED_LINK_IRQ));
2359 }
2360
2361 static void
2362 check_link_status(adapter_t *sc)
2363 {
2364         int i;
2365
2366         for (i = 0; i < (sc)->params.nports; ++i) {
2367                 struct port_info *p = &sc->port[i];
2368
2369                 if (!isset(&sc->open_device_map, p->port_id))
2370                         continue;
2371
2372                 if (link_poll_needed(p))
2373                         t3_link_changed(sc, i);
2374         }
2375 }
2376
2377 static void
2378 check_t3b2_mac(struct adapter *sc)
2379 {
2380         int i;
2381
2382         if (sc->flags & CXGB_SHUTDOWN)
2383                 return;
2384
2385         for_each_port(sc, i) {
2386                 struct port_info *p = &sc->port[i];
2387                 int status;
2388 #ifdef INVARIANTS
2389                 struct ifnet *ifp = p->ifp;
2390 #endif
2391
2392                 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2393                     !p->link_config.link_ok)
2394                         continue;
2395
2396                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2397                         ("%s: state mismatch (drv_flags %x, device_map %x)",
2398                          __func__, ifp->if_drv_flags, sc->open_device_map));
2399
2400                 PORT_LOCK(p);
2401                 status = t3b2_mac_watchdog_task(&p->mac);
2402                 if (status == 1)
2403                         p->mac.stats.num_toggled++;
2404                 else if (status == 2) {
2405                         struct cmac *mac = &p->mac;
2406
2407                         cxgb_update_mac_settings(p);
2408                         t3_link_start(&p->phy, mac, &p->link_config);
2409                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2410                         t3_port_intr_enable(sc, p->port_id);
2411                         p->mac.stats.num_resets++;
2412                 }
2413                 PORT_UNLOCK(p);
2414         }
2415 }
2416
2417 static void
2418 cxgb_tick(void *arg)
2419 {
2420         adapter_t *sc = (adapter_t *)arg;
2421
2422         if (sc->flags & CXGB_SHUTDOWN)
2423                 return;
2424
2425         taskqueue_enqueue(sc->tq, &sc->tick_task);
2426         callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
2427 }
2428
2429 static void
2430 cxgb_tick_handler(void *arg, int count)
2431 {
2432         adapter_t *sc = (adapter_t *)arg;
2433         const struct adapter_params *p = &sc->params;
2434         int i;
2435         uint32_t cause, reset;
2436
2437         if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2438                 return;
2439
2440         check_link_status(sc);
2441
2442         if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2443                 check_t3b2_mac(sc);
2444
2445         cause = t3_read_reg(sc, A_SG_INT_CAUSE);
2446         reset = 0;
2447         if (cause & F_FLEMPTY) {
2448                 struct sge_qset *qs = &sc->sge.qs[0];
2449
2450                 i = 0;
2451                 reset |= F_FLEMPTY;
2452
2453                 cause = (t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) >>
2454                          S_FL0EMPTY) & 0xffff;
2455                 while (cause) {
2456                         qs->fl[i].empty += (cause & 1);
2457                         if (i)
2458                                 qs++;
2459                         i ^= 1;
2460                         cause >>= 1;
2461                 }
2462         }
2463         t3_write_reg(sc, A_SG_INT_CAUSE, reset);
2464
2465         for (i = 0; i < sc->params.nports; i++) {
2466                 struct port_info *pi = &sc->port[i];
2467                 struct ifnet *ifp = pi->ifp;
2468                 struct cmac *mac = &pi->mac;
2469                 struct mac_stats *mstats = &mac->stats;
2470
2471                 if (!isset(&sc->open_device_map, pi->port_id))
2472                         continue;
2473
2474                 PORT_LOCK(pi);
2475                 t3_mac_update_stats(mac);
2476                 PORT_UNLOCK(pi);
2477
2478                 ifp->if_opackets =
2479                     mstats->tx_frames_64 +
2480                     mstats->tx_frames_65_127 +
2481                     mstats->tx_frames_128_255 +
2482                     mstats->tx_frames_256_511 +
2483                     mstats->tx_frames_512_1023 +
2484                     mstats->tx_frames_1024_1518 +
2485                     mstats->tx_frames_1519_max;
2486
2487                 ifp->if_ipackets =
2488                     mstats->rx_frames_64 +
2489                     mstats->rx_frames_65_127 +
2490                     mstats->rx_frames_128_255 +
2491                     mstats->rx_frames_256_511 +
2492                     mstats->rx_frames_512_1023 +
2493                     mstats->rx_frames_1024_1518 +
2494                     mstats->rx_frames_1519_max;
2495
2496                 ifp->if_obytes = mstats->tx_octets;
2497                 ifp->if_ibytes = mstats->rx_octets;
2498                 ifp->if_omcasts = mstats->tx_mcast_frames;
2499                 ifp->if_imcasts = mstats->rx_mcast_frames;
2500
2501                 ifp->if_collisions =
2502                     mstats->tx_total_collisions;
2503
2504                 ifp->if_iqdrops = mstats->rx_cong_drops;
2505
2506                 ifp->if_oerrors =
2507                     mstats->tx_excess_collisions +
2508                     mstats->tx_underrun +
2509                     mstats->tx_len_errs +
2510                     mstats->tx_mac_internal_errs +
2511                     mstats->tx_excess_deferral +
2512                     mstats->tx_fcs_errs;
2513                 ifp->if_ierrors =
2514                     mstats->rx_jabber +
2515                     mstats->rx_data_errs +
2516                     mstats->rx_sequence_errs +
2517                     mstats->rx_runt +
2518                     mstats->rx_too_long +
2519                     mstats->rx_mac_internal_errs +
2520                     mstats->rx_short +
2521                     mstats->rx_fcs_errs;
2522
2523                 if (mac->multiport)
2524                         continue;
2525
2526                 /* Count rx fifo overflows, once per second */
2527                 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2528                 reset = 0;
2529                 if (cause & F_RXFIFO_OVERFLOW) {
2530                         mac->stats.rx_fifo_ovfl++;
2531                         reset |= F_RXFIFO_OVERFLOW;
2532                 }
2533                 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2534         }
2535 }
2536
2537 static void
2538 touch_bars(device_t dev)
2539 {
2540         /*
2541          * Don't enable yet
2542          */
2543 #if !defined(__LP64__) && 0
2544         u32 v;
2545
2546         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2547         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2548         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2549         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2550         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2551         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2552 #endif
2553 }
2554
2555 static int
2556 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2557 {
2558         uint8_t *buf;
2559         int err = 0;
2560         u32 aligned_offset, aligned_len, *p;
2561         struct adapter *adapter = pi->adapter;
2562
2563
2564         aligned_offset = offset & ~3;
2565         aligned_len = (len + (offset & 3) + 3) & ~3;
2566
2567         if (aligned_offset != offset || aligned_len != len) {
2568                 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2569                 if (!buf)
2570                         return (ENOMEM);
2571                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2572                 if (!err && aligned_len > 4)
2573                         err = t3_seeprom_read(adapter,
2574                                               aligned_offset + aligned_len - 4,
2575                                               (u32 *)&buf[aligned_len - 4]);
2576                 if (err)
2577                         goto out;
2578                 memcpy(buf + (offset & 3), data, len);
2579         } else
2580                 buf = (uint8_t *)(uintptr_t)data;
2581
2582         err = t3_seeprom_wp(adapter, 0);
2583         if (err)
2584                 goto out;
2585
2586         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2587                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2588                 aligned_offset += 4;
2589         }
2590
2591         if (!err)
2592                 err = t3_seeprom_wp(adapter, 1);
2593 out:
2594         if (buf != data)
2595                 free(buf, M_DEVBUF);
2596         return err;
2597 }
2598
2599
2600 static int
2601 in_range(int val, int lo, int hi)
2602 {
2603         return val < 0 || (val <= hi && val >= lo);
2604 }
2605
2606 static int
2607 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2608 {
2609        return (0);
2610 }
2611
2612 static int
2613 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2614 {
2615        return (0);
2616 }
2617
2618 static int
2619 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2620     int fflag, struct thread *td)
2621 {
2622         int mmd, error = 0;
2623         struct port_info *pi = dev->si_drv1;
2624         adapter_t *sc = pi->adapter;
2625
2626 #ifdef PRIV_SUPPORTED
2627         if (priv_check(td, PRIV_DRIVER)) {
2628                 if (cxgb_debug)
2629                         printf("user does not have access to privileged ioctls\n");
2630                 return (EPERM);
2631         }
2632 #else
2633         if (suser(td)) {
2634                 if (cxgb_debug)
2635                         printf("user does not have access to privileged ioctls\n");
2636                 return (EPERM);
2637         }
2638 #endif
2639
2640         switch (cmd) {
2641         case CHELSIO_GET_MIIREG: {
2642                 uint32_t val;
2643                 struct cphy *phy = &pi->phy;
2644                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2645
2646                 if (!phy->mdio_read)
2647                         return (EOPNOTSUPP);
2648                 if (is_10G(sc)) {
2649                         mmd = mid->phy_id >> 8;
2650                         if (!mmd)
2651                                 mmd = MDIO_DEV_PCS;
2652                         else if (mmd > MDIO_DEV_VEND2)
2653                                 return (EINVAL);
2654
2655                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2656                                              mid->reg_num, &val);
2657                 } else
2658                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2659                                              mid->reg_num & 0x1f, &val);
2660                 if (error == 0)
2661                         mid->val_out = val;
2662                 break;
2663         }
2664         case CHELSIO_SET_MIIREG: {
2665                 struct cphy *phy = &pi->phy;
2666                 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2667
2668                 if (!phy->mdio_write)
2669                         return (EOPNOTSUPP);
2670                 if (is_10G(sc)) {
2671                         mmd = mid->phy_id >> 8;
2672                         if (!mmd)
2673                                 mmd = MDIO_DEV_PCS;
2674                         else if (mmd > MDIO_DEV_VEND2)
2675                                 return (EINVAL);
2676
2677                         error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2678                                               mmd, mid->reg_num, mid->val_in);
2679                 } else
2680                         error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2681                                               mid->reg_num & 0x1f,
2682                                               mid->val_in);
2683                 break;
2684         }
2685         case CHELSIO_SETREG: {
2686                 struct ch_reg *edata = (struct ch_reg *)data;
2687                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2688                         return (EFAULT);
2689                 t3_write_reg(sc, edata->addr, edata->val);
2690                 break;
2691         }
2692         case CHELSIO_GETREG: {
2693                 struct ch_reg *edata = (struct ch_reg *)data;
2694                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2695                         return (EFAULT);
2696                 edata->val = t3_read_reg(sc, edata->addr);
2697                 break;
2698         }
2699         case CHELSIO_GET_SGE_CONTEXT: {
2700                 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2701                 mtx_lock_spin(&sc->sge.reg_lock);
2702                 switch (ecntxt->cntxt_type) {
2703                 case CNTXT_TYPE_EGRESS:
2704                         error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2705                             ecntxt->data);
2706                         break;
2707                 case CNTXT_TYPE_FL:
2708                         error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2709                             ecntxt->data);
2710                         break;
2711                 case CNTXT_TYPE_RSP:
2712                         error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2713                             ecntxt->data);
2714                         break;
2715                 case CNTXT_TYPE_CQ:
2716                         error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2717                             ecntxt->data);
2718                         break;
2719                 default:
2720                         error = EINVAL;
2721                         break;
2722                 }
2723                 mtx_unlock_spin(&sc->sge.reg_lock);
2724                 break;
2725         }
2726         case CHELSIO_GET_SGE_DESC: {
2727                 struct ch_desc *edesc = (struct ch_desc *)data;
2728                 int ret;
2729                 if (edesc->queue_num >= SGE_QSETS * 6)
2730                         return (EINVAL);
2731                 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2732                     edesc->queue_num % 6, edesc->idx, edesc->data);
2733                 if (ret < 0)
2734                         return (EINVAL);
2735                 edesc->size = ret;
2736                 break;
2737         }
2738         case CHELSIO_GET_QSET_PARAMS: {
2739                 struct qset_params *q;
2740                 struct ch_qset_params *t = (struct ch_qset_params *)data;
2741                 int q1 = pi->first_qset;
2742                 int nqsets = pi->nqsets;
2743                 int i;
2744
2745                 if (t->qset_idx >= nqsets)
2746                         return EINVAL;
2747
2748                 i = q1 + t->qset_idx;
2749                 q = &sc->params.sge.qset[i];
2750                 t->rspq_size   = q->rspq_size;
2751                 t->txq_size[0] = q->txq_size[0];
2752                 t->txq_size[1] = q->txq_size[1];
2753                 t->txq_size[2] = q->txq_size[2];
2754                 t->fl_size[0]  = q->fl_size;
2755                 t->fl_size[1]  = q->jumbo_size;
2756                 t->polling     = q->polling;
2757                 t->lro         = q->lro;
2758                 t->intr_lat    = q->coalesce_usecs;
2759                 t->cong_thres  = q->cong_thres;
2760                 t->qnum        = i;
2761
2762                 if (sc->flags & USING_MSIX)
2763                         t->vector = rman_get_start(sc->msix_irq_res[i]);
2764                 else
2765                         t->vector = rman_get_start(sc->irq_res);
2766
2767                 break;
2768         }
2769         case CHELSIO_GET_QSET_NUM: {
2770                 struct ch_reg *edata = (struct ch_reg *)data;
2771                 edata->val = pi->nqsets;
2772                 break;
2773         }
2774         case CHELSIO_LOAD_FW: {
2775                 uint8_t *fw_data;
2776                 uint32_t vers;
2777                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2778
2779                 /*
2780                  * You're allowed to load a firmware only before FULL_INIT_DONE
2781                  *
2782                  * FW_UPTODATE is also set so the rest of the initialization
2783                  * will not overwrite what was loaded here.  This gives you the
2784                  * flexibility to load any firmware (and maybe shoot yourself in
2785                  * the foot).
2786                  */
2787
2788                 ADAPTER_LOCK(sc);
2789                 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2790                         ADAPTER_UNLOCK(sc);
2791                         return (EBUSY);
2792                 }
2793
2794                 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2795                 if (!fw_data)
2796                         error = ENOMEM;
2797                 else
2798                         error = copyin(t->buf, fw_data, t->len);
2799
2800                 if (!error)
2801                         error = -t3_load_fw(sc, fw_data, t->len);
2802
2803                 if (t3_get_fw_version(sc, &vers) == 0) {
2804                         snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2805                             "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2806                             G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2807                 }
2808
2809                 if (!error)
2810                         sc->flags |= FW_UPTODATE;
2811
2812                 free(fw_data, M_DEVBUF);
2813                 ADAPTER_UNLOCK(sc);
2814                 break;
2815         }
2816         case CHELSIO_LOAD_BOOT: {
2817                 uint8_t *boot_data;
2818                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2819
2820                 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2821                 if (!boot_data)
2822                         return ENOMEM;
2823
2824                 error = copyin(t->buf, boot_data, t->len);
2825                 if (!error)
2826                         error = -t3_load_boot(sc, boot_data, t->len);
2827
2828                 free(boot_data, M_DEVBUF);
2829                 break;
2830         }
2831         case CHELSIO_GET_PM: {
2832                 struct ch_pm *m = (struct ch_pm *)data;
2833                 struct tp_params *p = &sc->params.tp;
2834
2835                 if (!is_offload(sc))
2836                         return (EOPNOTSUPP);
2837
2838                 m->tx_pg_sz = p->tx_pg_size;
2839                 m->tx_num_pg = p->tx_num_pgs;
2840                 m->rx_pg_sz  = p->rx_pg_size;
2841                 m->rx_num_pg = p->rx_num_pgs;
2842                 m->pm_total  = p->pmtx_size + p->chan_rx_size * p->nchan;
2843
2844                 break;
2845         }
2846         case CHELSIO_SET_PM: {
2847                 struct ch_pm *m = (struct ch_pm *)data;
2848                 struct tp_params *p = &sc->params.tp;
2849
2850                 if (!is_offload(sc))
2851                         return (EOPNOTSUPP);
2852                 if (sc->flags & FULL_INIT_DONE)
2853                         return (EBUSY);
2854
2855                 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2856                     !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2857                         return (EINVAL);        /* not power of 2 */
2858                 if (!(m->rx_pg_sz & 0x14000))
2859                         return (EINVAL);        /* not 16KB or 64KB */
2860                 if (!(m->tx_pg_sz & 0x1554000))
2861                         return (EINVAL);
2862                 if (m->tx_num_pg == -1)
2863                         m->tx_num_pg = p->tx_num_pgs;
2864                 if (m->rx_num_pg == -1)
2865                         m->rx_num_pg = p->rx_num_pgs;
2866                 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2867                         return (EINVAL);
2868                 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2869                     m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2870                         return (EINVAL);
2871
2872                 p->rx_pg_size = m->rx_pg_sz;
2873                 p->tx_pg_size = m->tx_pg_sz;
2874                 p->rx_num_pgs = m->rx_num_pg;
2875                 p->tx_num_pgs = m->tx_num_pg;
2876                 break;
2877         }
2878         case CHELSIO_SETMTUTAB: {
2879                 struct ch_mtus *m = (struct ch_mtus *)data;
2880                 int i;
2881
2882                 if (!is_offload(sc))
2883                         return (EOPNOTSUPP);
2884                 if (offload_running(sc))
2885                         return (EBUSY);
2886                 if (m->nmtus != NMTUS)
2887                         return (EINVAL);
2888                 if (m->mtus[0] < 81)         /* accommodate SACK */
2889                         return (EINVAL);
2890
2891                 /*
2892                  * MTUs must be in ascending order
2893                  */
2894                 for (i = 1; i < NMTUS; ++i)
2895                         if (m->mtus[i] < m->mtus[i - 1])
2896                                 return (EINVAL);
2897
2898                 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2899                 break;
2900         }
2901         case CHELSIO_GETMTUTAB: {
2902                 struct ch_mtus *m = (struct ch_mtus *)data;
2903
2904                 if (!is_offload(sc))
2905                         return (EOPNOTSUPP);
2906
2907                 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2908                 m->nmtus = NMTUS;
2909                 break;
2910         }
2911         case CHELSIO_GET_MEM: {
2912                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2913                 struct mc7 *mem;
2914                 uint8_t *useraddr;
2915                 u64 buf[32];
2916
2917                 /*
2918                  * Use these to avoid modifying len/addr in the the return
2919                  * struct
2920                  */
2921                 uint32_t len = t->len, addr = t->addr;
2922
2923                 if (!is_offload(sc))
2924                         return (EOPNOTSUPP);
2925                 if (!(sc->flags & FULL_INIT_DONE))
2926                         return (EIO);         /* need the memory controllers */
2927                 if ((addr & 0x7) || (len & 0x7))
2928                         return (EINVAL);
2929                 if (t->mem_id == MEM_CM)
2930                         mem = &sc->cm;
2931                 else if (t->mem_id == MEM_PMRX)
2932                         mem = &sc->pmrx;
2933                 else if (t->mem_id == MEM_PMTX)
2934                         mem = &sc->pmtx;
2935                 else
2936                         return (EINVAL);
2937
2938                 /*
2939                  * Version scheme:
2940                  * bits 0..9: chip version
2941                  * bits 10..15: chip revision
2942                  */
2943                 t->version = 3 | (sc->params.rev << 10);
2944
2945                 /*
2946                  * Read 256 bytes at a time as len can be large and we don't
2947                  * want to use huge intermediate buffers.
2948                  */
2949                 useraddr = (uint8_t *)t->buf;
2950                 while (len) {
2951                         unsigned int chunk = min(len, sizeof(buf));
2952
2953                         error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2954                         if (error)
2955                                 return (-error);
2956                         if (copyout(buf, useraddr, chunk))
2957                                 return (EFAULT);
2958                         useraddr += chunk;
2959                         addr += chunk;
2960                         len -= chunk;
2961                 }
2962                 break;
2963         }
2964         case CHELSIO_READ_TCAM_WORD: {
2965                 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2966
2967                 if (!is_offload(sc))
2968                         return (EOPNOTSUPP);
2969                 if (!(sc->flags & FULL_INIT_DONE))
2970                         return (EIO);         /* need MC5 */
2971                 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2972                 break;
2973         }
2974         case CHELSIO_SET_TRACE_FILTER: {
2975                 struct ch_trace *t = (struct ch_trace *)data;
2976                 const struct trace_params *tp;
2977
2978                 tp = (const struct trace_params *)&t->sip;
2979                 if (t->config_tx)
2980                         t3_config_trace_filter(sc, tp, 0, t->invert_match,
2981                                                t->trace_tx);
2982                 if (t->config_rx)
2983                         t3_config_trace_filter(sc, tp, 1, t->invert_match,
2984                                                t->trace_rx);
2985                 break;
2986         }
2987         case CHELSIO_SET_PKTSCHED: {
2988                 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2989                 if (sc->open_device_map == 0)
2990                         return (EAGAIN);
2991                 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2992                     p->binding);
2993                 break;
2994         }
2995         case CHELSIO_IFCONF_GETREGS: {
2996                 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2997                 int reglen = cxgb_get_regs_len();
2998                 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2999                 if (buf == NULL) {
3000                         return (ENOMEM);
3001                 }
3002                 if (regs->len > reglen)
3003                         regs->len = reglen;
3004                 else if (regs->len < reglen)
3005                         error = ENOBUFS;
3006
3007                 if (!error) {
3008                         cxgb_get_regs(sc, regs, buf);
3009                         error = copyout(buf, regs->data, reglen);
3010                 }
3011                 free(buf, M_DEVBUF);
3012
3013                 break;
3014         }
3015         case CHELSIO_SET_HW_SCHED: {
3016                 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
3017                 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
3018
3019                 if ((sc->flags & FULL_INIT_DONE) == 0)
3020                         return (EAGAIN);       /* need TP to be initialized */
3021                 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
3022                     !in_range(t->channel, 0, 1) ||
3023                     !in_range(t->kbps, 0, 10000000) ||
3024                     !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
3025                     !in_range(t->flow_ipg, 0,
3026                               dack_ticks_to_usec(sc, 0x7ff)))
3027                         return (EINVAL);
3028
3029                 if (t->kbps >= 0) {
3030                         error = t3_config_sched(sc, t->kbps, t->sched);
3031                         if (error < 0)
3032                                 return (-error);
3033                 }
3034                 if (t->class_ipg >= 0)
3035                         t3_set_sched_ipg(sc, t->sched, t->class_ipg);
3036                 if (t->flow_ipg >= 0) {
3037                         t->flow_ipg *= 1000;     /* us -> ns */
3038                         t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
3039                 }
3040                 if (t->mode >= 0) {
3041                         int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
3042
3043                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
3044                                          bit, t->mode ? bit : 0);
3045                 }
3046                 if (t->channel >= 0)
3047                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
3048                                          1 << t->sched, t->channel << t->sched);
3049                 break;
3050         }
3051         case CHELSIO_GET_EEPROM: {
3052                 int i;
3053                 struct ch_eeprom *e = (struct ch_eeprom *)data;
3054                 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
3055
3056                 if (buf == NULL) {
3057                         return (ENOMEM);
3058                 }
3059                 e->magic = EEPROM_MAGIC;
3060                 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
3061                         error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
3062
3063                 if (!error)
3064                         error = copyout(buf + e->offset, e->data, e->len);
3065
3066                 free(buf, M_DEVBUF);
3067                 break;
3068         }
3069         case CHELSIO_CLEAR_STATS: {
3070                 if (!(sc->flags & FULL_INIT_DONE))
3071                         return EAGAIN;
3072
3073                 PORT_LOCK(pi);
3074                 t3_mac_update_stats(&pi->mac);
3075                 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
3076                 PORT_UNLOCK(pi);
3077                 break;
3078         }
3079         case CHELSIO_GET_UP_LA: {
3080                 struct ch_up_la *la = (struct ch_up_la *)data;
3081                 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
3082                 if (buf == NULL) {
3083                         return (ENOMEM);
3084                 }
3085                 if (la->bufsize < LA_BUFSIZE)
3086                         error = ENOBUFS;
3087
3088                 if (!error)
3089                         error = -t3_get_up_la(sc, &la->stopped, &la->idx,
3090                                               &la->bufsize, buf);
3091                 if (!error)
3092                         error = copyout(buf, la->data, la->bufsize);
3093
3094                 free(buf, M_DEVBUF);
3095                 break;
3096         }
3097         case CHELSIO_GET_UP_IOQS: {
3098                 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3099                 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3100                 uint32_t *v;
3101
3102                 if (buf == NULL) {
3103                         return (ENOMEM);
3104                 }
3105                 if (ioqs->bufsize < IOQS_BUFSIZE)
3106                         error = ENOBUFS;
3107
3108                 if (!error)
3109                         error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3110
3111                 if (!error) {
3112                         v = (uint32_t *)buf;
3113
3114                         ioqs->bufsize -= 4 * sizeof(uint32_t);
3115                         ioqs->ioq_rx_enable = *v++;
3116                         ioqs->ioq_tx_enable = *v++;
3117                         ioqs->ioq_rx_status = *v++;
3118                         ioqs->ioq_tx_status = *v++;
3119
3120                         error = copyout(v, ioqs->data, ioqs->bufsize);
3121                 }
3122
3123                 free(buf, M_DEVBUF);
3124                 break;
3125         }
3126         default:
3127                 return (EOPNOTSUPP);
3128                 break;
3129         }
3130
3131         return (error);
3132 }
3133
3134 static __inline void
3135 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3136     unsigned int end)
3137 {
3138         uint32_t *p = (uint32_t *)(buf + start);
3139
3140         for ( ; start <= end; start += sizeof(uint32_t))
3141                 *p++ = t3_read_reg(ap, start);
3142 }
3143
3144 #define T3_REGMAP_SIZE (3 * 1024)
3145 static int
3146 cxgb_get_regs_len(void)
3147 {
3148         return T3_REGMAP_SIZE;
3149 }
3150
3151 static void
3152 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3153 {
3154
3155         /*
3156          * Version scheme:
3157          * bits 0..9: chip version
3158          * bits 10..15: chip revision
3159          * bit 31: set for PCIe cards
3160          */
3161         regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3162
3163         /*
3164          * We skip the MAC statistics registers because they are clear-on-read.
3165          * Also reading multi-register stats would need to synchronize with the
3166          * periodic mac stats accumulation.  Hard to justify the complexity.
3167          */
3168         memset(buf, 0, cxgb_get_regs_len());
3169         reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3170         reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3171         reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3172         reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3173         reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3174         reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3175                        XGM_REG(A_XGM_SERDES_STAT3, 1));
3176         reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3177                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3178 }
3179
3180
3181 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1);