1 /**************************************************************************
3 Copyright (c) 2007-2009, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/pciio.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/bus_dma.h>
45 #include <sys/ioccom.h>
47 #include <sys/linker.h>
48 #include <sys/firmware.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/queue.h>
55 #include <sys/taskqueue.h>
59 #include <net/ethernet.h>
61 #include <net/if_arp.h>
62 #include <net/if_dl.h>
63 #include <net/if_media.h>
64 #include <net/if_types.h>
65 #include <net/if_vlan_var.h>
67 #include <netinet/in_systm.h>
68 #include <netinet/in.h>
69 #include <netinet/if_ether.h>
70 #include <netinet/ip.h>
71 #include <netinet/ip.h>
72 #include <netinet/tcp.h>
73 #include <netinet/udp.h>
75 #include <dev/pci/pcireg.h>
76 #include <dev/pci/pcivar.h>
77 #include <dev/pci/pci_private.h>
79 #include <cxgb_include.h>
85 static int cxgb_setup_interrupts(adapter_t *);
86 static void cxgb_teardown_interrupts(adapter_t *);
87 static int cxgb_begin_op(struct port_info *, const char *);
88 static int cxgb_begin_detach(struct port_info *);
89 static int cxgb_end_op(struct port_info *);
90 static void cxgb_init(void *);
91 static int cxgb_init_synchronized(struct port_info *);
92 static int cxgb_uninit_synchronized(struct port_info *);
93 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
94 static int cxgb_media_change(struct ifnet *);
95 static int cxgb_ifm_type(int);
96 static void cxgb_build_medialist(struct port_info *);
97 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
98 static int setup_sge_qsets(adapter_t *);
99 static void cxgb_async_intr(void *);
100 static void cxgb_ext_intr_handler(void *, int);
101 static void cxgb_tick_handler(void *, int);
102 static void cxgb_tick(void *);
103 static void setup_rss(adapter_t *sc);
105 /* Attachment glue for the PCI controller end of the device. Each port of
106 * the device is attached separately, as defined later.
108 static int cxgb_controller_probe(device_t);
109 static int cxgb_controller_attach(device_t);
110 static int cxgb_controller_detach(device_t);
111 static void cxgb_free(struct adapter *);
112 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
114 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
115 static int cxgb_get_regs_len(void);
116 static int offload_open(struct port_info *pi);
117 static void touch_bars(device_t dev);
118 static int offload_close(struct t3cdev *tdev);
119 int t3_detect_link_fault(adapter_t *adapter, int port_id);
121 static device_method_t cxgb_controller_methods[] = {
122 DEVMETHOD(device_probe, cxgb_controller_probe),
123 DEVMETHOD(device_attach, cxgb_controller_attach),
124 DEVMETHOD(device_detach, cxgb_controller_detach),
127 DEVMETHOD(bus_print_child, bus_generic_print_child),
128 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
133 static driver_t cxgb_controller_driver = {
135 cxgb_controller_methods,
136 sizeof(struct adapter)
139 static devclass_t cxgb_controller_devclass;
140 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
143 * Attachment glue for the ports. Attachment is done directly to the
146 static int cxgb_port_probe(device_t);
147 static int cxgb_port_attach(device_t);
148 static int cxgb_port_detach(device_t);
150 static device_method_t cxgb_port_methods[] = {
151 DEVMETHOD(device_probe, cxgb_port_probe),
152 DEVMETHOD(device_attach, cxgb_port_attach),
153 DEVMETHOD(device_detach, cxgb_port_detach),
157 static driver_t cxgb_port_driver = {
163 static d_ioctl_t cxgb_extension_ioctl;
164 static d_open_t cxgb_extension_open;
165 static d_close_t cxgb_extension_close;
167 static struct cdevsw cxgb_cdevsw = {
168 .d_version = D_VERSION,
170 .d_open = cxgb_extension_open,
171 .d_close = cxgb_extension_close,
172 .d_ioctl = cxgb_extension_ioctl,
176 static devclass_t cxgb_port_devclass;
177 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
180 * The driver uses the best interrupt scheme available on a platform in the
181 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
182 * of these schemes the driver may consider as follows:
184 * msi = 2: choose from among all three options
185 * msi = 1 : only consider MSI and pin interrupts
186 * msi = 0: force pin interrupts
188 static int msi_allowed = 2;
190 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
191 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
192 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
193 "MSI-X, MSI, INTx selector");
196 * The driver enables offload as a default.
197 * To disable it, use ofld_disable = 1.
199 static int ofld_disable = 0;
200 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
201 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
202 "disable ULP offload");
205 * The driver uses an auto-queue algorithm by default.
206 * To disable it and force a single queue-set per port, use multiq = 0
208 static int multiq = 1;
209 TUNABLE_INT("hw.cxgb.multiq", &multiq);
210 SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
211 "use min(ncpus/ports, 8) queue-sets per port");
214 * By default the driver will not update the firmware unless
215 * it was compiled against a newer version
218 static int force_fw_update = 0;
219 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
220 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
221 "update firmware even if up to date");
223 int cxgb_use_16k_clusters = 1;
224 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
225 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
226 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
229 * Tune the size of the output queue.
231 int cxgb_snd_queue_len = IFQ_MAXLEN;
232 TUNABLE_INT("hw.cxgb.snd_queue_len", &cxgb_snd_queue_len);
233 SYSCTL_UINT(_hw_cxgb, OID_AUTO, snd_queue_len, CTLFLAG_RDTUN,
234 &cxgb_snd_queue_len, 0, "send queue size ");
238 MAX_TXQ_ENTRIES = 16384,
239 MAX_CTRL_TXQ_ENTRIES = 1024,
240 MAX_RSPQ_ENTRIES = 16384,
241 MAX_RX_BUFFERS = 16384,
242 MAX_RX_JUMBO_BUFFERS = 16384,
244 MIN_CTRL_TXQ_ENTRIES = 4,
245 MIN_RSPQ_ENTRIES = 32,
247 MIN_FL_JUMBO_ENTRIES = 32
262 u32 report_filter_id:1;
270 enum { FILTER_NO_VLAN_PRI = 7 };
272 #define EEPROM_MAGIC 0x38E2F10C
274 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
276 /* Table for probing the cards. The desc field isn't actually used */
282 } cxgb_identifiers[] = {
283 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
284 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
285 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
286 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
287 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
288 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
289 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
290 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
291 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
292 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
293 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
294 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "N310E"},
298 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
302 t3rev2char(struct adapter *adapter)
306 switch(adapter->params.rev) {
321 static struct cxgb_ident *
322 cxgb_get_ident(device_t dev)
324 struct cxgb_ident *id;
326 for (id = cxgb_identifiers; id->desc != NULL; id++) {
327 if ((id->vendor == pci_get_vendor(dev)) &&
328 (id->device == pci_get_device(dev))) {
335 static const struct adapter_info *
336 cxgb_get_adapter_info(device_t dev)
338 struct cxgb_ident *id;
339 const struct adapter_info *ai;
341 id = cxgb_get_ident(dev);
345 ai = t3_get_adapter_info(id->index);
351 cxgb_controller_probe(device_t dev)
353 const struct adapter_info *ai;
354 char *ports, buf[80];
356 struct adapter *sc = device_get_softc(dev);
358 ai = cxgb_get_adapter_info(dev);
362 nports = ai->nports0 + ai->nports1;
368 snprintf(buf, sizeof(buf), "%s %sNIC, rev: %d nports: %d %s",
369 ai->desc, is_offload(sc) ? "R" : "",
370 sc->params.rev, nports, ports);
371 device_set_desc_copy(dev, buf);
372 return (BUS_PROBE_DEFAULT);
375 #define FW_FNAME "cxgb_t3fw"
376 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
377 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
380 upgrade_fw(adapter_t *sc)
382 #ifdef FIRMWARE_LATEST
383 const struct firmware *fw;
389 if ((fw = firmware_get(FW_FNAME)) == NULL) {
390 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
393 device_printf(sc->dev, "updating firmware on card\n");
394 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
396 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status);
398 firmware_put(fw, FIRMWARE_UNLOAD);
404 * The cxgb_controller_attach function is responsible for the initial
405 * bringup of the device. Its responsibilities include:
407 * 1. Determine if the device supports MSI or MSI-X.
408 * 2. Allocate bus resources so that we can access the Base Address Register
409 * 3. Create and initialize mutexes for the controller and its control
410 * logic such as SGE and MDIO.
411 * 4. Call hardware specific setup routine for the adapter as a whole.
412 * 5. Allocate the BAR for doing MSI-X.
413 * 6. Setup the line interrupt iff MSI-X is not supported.
414 * 7. Create the driver's taskq.
415 * 8. Start one task queue service thread.
416 * 9. Check if the firmware and SRAM are up-to-date. They will be
417 * auto-updated later (before FULL_INIT_DONE), if required.
418 * 10. Create a child device for each MAC (port)
419 * 11. Initialize T3 private state.
420 * 12. Trigger the LED
421 * 13. Setup offload iff supported.
422 * 14. Reset/restart the tick callout.
425 * NOTE: Any modification or deviation from this list MUST be reflected in
426 * the above comment. Failure to do so will result in problems on various
427 * error conditions including link flapping.
430 cxgb_controller_attach(device_t dev)
433 const struct adapter_info *ai;
443 sc = device_get_softc(dev);
446 ai = cxgb_get_adapter_info(dev);
449 * XXX not really related but a recent addition
452 /* find the PCIe link width and set max read request to 4KB*/
453 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
455 lnk = pci_read_config(dev, reg + 0x12, 2);
456 sc->link_width = (lnk >> 4) & 0x3f;
458 pectl = pci_read_config(dev, reg + 0x8, 2);
459 pectl = (pectl & ~0x7000) | (5 << 12);
460 pci_write_config(dev, reg + 0x8, pectl, 2);
463 if (sc->link_width != 0 && sc->link_width <= 4 &&
464 (ai->nports0 + ai->nports1) <= 2) {
465 device_printf(sc->dev,
466 "PCIe x%d Link, expect reduced performance\n",
471 pci_enable_busmaster(dev);
473 * Allocate the registers and make them available to the driver.
474 * The registers that we care about for NIC mode are in BAR 0
476 sc->regs_rid = PCIR_BAR(0);
477 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
478 &sc->regs_rid, RF_ACTIVE)) == NULL) {
479 device_printf(dev, "Cannot allocate BAR region 0\n");
482 sc->udbs_rid = PCIR_BAR(2);
484 if (is_offload(sc) &&
485 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
486 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
487 device_printf(dev, "Cannot allocate BAR region 1\n");
492 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
493 device_get_unit(dev));
494 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
496 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
497 device_get_unit(dev));
498 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
499 device_get_unit(dev));
500 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
501 device_get_unit(dev));
503 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
504 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
505 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
507 sc->bt = rman_get_bustag(sc->regs_res);
508 sc->bh = rman_get_bushandle(sc->regs_res);
509 sc->mmio_len = rman_get_size(sc->regs_res);
511 if (t3_prep_adapter(sc, ai, 1) < 0) {
512 printf("prep adapter failed\n");
516 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
517 * enough messages for the queue sets. If that fails, try falling
518 * back to MSI. If that fails, then try falling back to the legacy
519 * interrupt pin model.
523 sc->msix_regs_rid = 0x20;
524 if ((msi_allowed >= 2) &&
525 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
526 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
529 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
530 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
532 if (pci_msix_count(dev) == 0 ||
533 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
534 sc->msi_count != msi_needed) {
535 device_printf(dev, "alloc msix failed - "
536 "msi_count=%d, msi_needed=%d, err=%d; "
537 "will try MSI\n", sc->msi_count,
541 pci_release_msi(dev);
542 bus_release_resource(dev, SYS_RES_MEMORY,
543 sc->msix_regs_rid, sc->msix_regs_res);
544 sc->msix_regs_res = NULL;
546 sc->flags |= USING_MSIX;
547 sc->cxgb_intr = cxgb_async_intr;
549 "using MSI-X interrupts (%u vectors)\n",
554 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
556 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
557 device_printf(dev, "alloc msi failed - "
558 "err=%d; will try INTx\n", error);
561 pci_release_msi(dev);
563 sc->flags |= USING_MSI;
564 sc->cxgb_intr = t3_intr_msi;
565 device_printf(dev, "using MSI interrupts\n");
569 if (sc->msi_count == 0) {
570 device_printf(dev, "using line interrupts\n");
571 sc->cxgb_intr = t3b_intr;
574 /* Create a private taskqueue thread for handling driver events */
575 #ifdef TASKQUEUE_CURRENT
576 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
577 taskqueue_thread_enqueue, &sc->tq);
579 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
580 taskqueue_thread_enqueue, &sc->tq);
582 if (sc->tq == NULL) {
583 device_printf(dev, "failed to allocate controller task queue\n");
587 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
588 device_get_nameunit(dev));
589 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
590 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
593 /* Create a periodic callout for checking adapter status */
594 callout_init(&sc->cxgb_tick_ch, TRUE);
596 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
598 * Warn user that a firmware update will be attempted in init.
600 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
601 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
602 sc->flags &= ~FW_UPTODATE;
604 sc->flags |= FW_UPTODATE;
607 if (t3_check_tpsram_version(sc) < 0) {
609 * Warn user that a firmware update will be attempted in init.
611 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
612 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
613 sc->flags &= ~TPS_UPTODATE;
615 sc->flags |= TPS_UPTODATE;
619 * Create a child device for each MAC. The ethernet attachment
620 * will be done in these children.
622 for (i = 0; i < (sc)->params.nports; i++) {
623 struct port_info *pi;
625 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
626 device_printf(dev, "failed to add child port\n");
632 pi->nqsets = port_qsets;
633 pi->first_qset = i*port_qsets;
635 pi->tx_chan = i >= ai->nports0;
636 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
637 sc->rxpkt_map[pi->txpkt_intf] = i;
638 sc->port[i].tx_chan = i >= ai->nports0;
639 sc->portdev[i] = child;
640 device_set_softc(child, pi);
642 if ((error = bus_generic_attach(dev)) != 0)
645 /* initialize sge private state */
646 t3_sge_init_adapter(sc);
651 if (is_offload(sc)) {
652 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
653 cxgb_adapter_ofld(sc);
655 error = t3_get_fw_version(sc, &vers);
659 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
660 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
661 G_FW_VERSION_MICRO(vers));
663 snprintf(buf, sizeof(buf), "%s\t E/C: %s S/N: %s",
665 sc->params.vpd.ec, sc->params.vpd.sn);
666 device_set_desc_copy(dev, buf);
668 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
669 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
670 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
672 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
673 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
674 t3_add_attach_sysctls(sc);
683 * The cxgb_controller_detach routine is called with the device is
684 * unloaded from the system.
688 cxgb_controller_detach(device_t dev)
692 sc = device_get_softc(dev);
700 * The cxgb_free() is called by the cxgb_controller_detach() routine
701 * to tear down the structures that were built up in
702 * cxgb_controller_attach(), and should be the final piece of work
703 * done when fully unloading the driver.
706 * 1. Shutting down the threads started by the cxgb_controller_attach()
708 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
709 * 3. Detaching all of the port devices created during the
710 * cxgb_controller_attach() routine.
711 * 4. Removing the device children created via cxgb_controller_attach().
712 * 5. Releasing PCI resources associated with the device.
713 * 6. Turning off the offload support, iff it was turned on.
714 * 7. Destroying the mutexes created in cxgb_controller_attach().
718 cxgb_free(struct adapter *sc)
723 sc->flags |= CXGB_SHUTDOWN;
727 * Make sure all child devices are gone.
729 bus_generic_detach(sc->dev);
730 for (i = 0; i < (sc)->params.nports; i++) {
731 if (sc->portdev[i] &&
732 device_delete_child(sc->dev, sc->portdev[i]) != 0)
733 device_printf(sc->dev, "failed to delete child port\n");
737 * At this point, it is as if cxgb_port_detach has run on all ports, and
738 * cxgb_down has run on the adapter. All interrupts have been silenced,
739 * all open devices have been closed.
741 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
742 __func__, sc->open_device_map));
743 for (i = 0; i < sc->params.nports; i++) {
744 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
749 * Finish off the adapter's callouts.
751 callout_drain(&sc->cxgb_tick_ch);
752 callout_drain(&sc->sge_timer_ch);
755 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
756 * sysctls are cleaned up by the kernel linker.
758 if (sc->flags & FULL_INIT_DONE) {
759 t3_free_sge_resources(sc);
760 sc->flags &= ~FULL_INIT_DONE;
764 * Release all interrupt resources.
766 cxgb_teardown_interrupts(sc);
768 if (sc->flags & (USING_MSI | USING_MSIX)) {
769 device_printf(sc->dev, "releasing msi message(s)\n");
770 pci_release_msi(sc->dev);
772 device_printf(sc->dev, "no msi message to release\n");
775 if (sc->msix_regs_res != NULL) {
776 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
782 * Free the adapter's taskqueue.
784 if (sc->tq != NULL) {
785 taskqueue_free(sc->tq);
789 if (is_offload(sc)) {
790 clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
791 cxgb_adapter_unofld(sc);
795 if (sc->flags & CXGB_OFLD_INIT)
796 cxgb_offload_deactivate(sc);
798 free(sc->filters, M_DEVBUF);
803 if (sc->udbs_res != NULL)
804 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
807 if (sc->regs_res != NULL)
808 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
811 MTX_DESTROY(&sc->mdio_lock);
812 MTX_DESTROY(&sc->sge.reg_lock);
813 MTX_DESTROY(&sc->elmer_lock);
814 ADAPTER_LOCK_DEINIT(sc);
818 * setup_sge_qsets - configure SGE Tx/Rx/response queues
819 * @sc: the controller softc
821 * Determines how many sets of SGE queues to use and initializes them.
822 * We support multiple queue sets per port if we have MSI-X, otherwise
823 * just one queue set per port.
826 setup_sge_qsets(adapter_t *sc)
828 int i, j, err, irq_idx = 0, qset_idx = 0;
829 u_int ntxq = SGE_TXQ_PER_SET;
831 if ((err = t3_sge_alloc(sc)) != 0) {
832 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
836 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
839 for (i = 0; i < (sc)->params.nports; i++) {
840 struct port_info *pi = &sc->port[i];
842 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
843 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
844 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
845 &sc->params.sge.qset[qset_idx], ntxq, pi);
847 t3_free_sge_resources(sc);
848 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n",
859 cxgb_teardown_interrupts(adapter_t *sc)
863 for (i = 0; i < SGE_QSETS; i++) {
864 if (sc->msix_intr_tag[i] == NULL) {
866 /* Should have been setup fully or not at all */
867 KASSERT(sc->msix_irq_res[i] == NULL &&
868 sc->msix_irq_rid[i] == 0,
869 ("%s: half-done interrupt (%d).", __func__, i));
874 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
875 sc->msix_intr_tag[i]);
876 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
877 sc->msix_irq_res[i]);
879 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
880 sc->msix_irq_rid[i] = 0;
884 KASSERT(sc->irq_res != NULL,
885 ("%s: half-done interrupt.", __func__));
887 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
888 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
891 sc->irq_res = sc->intr_tag = NULL;
897 cxgb_setup_interrupts(adapter_t *sc)
899 struct resource *res;
901 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
903 sc->irq_rid = intr_flag ? 1 : 0;
904 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
905 RF_SHAREABLE | RF_ACTIVE);
906 if (sc->irq_res == NULL) {
907 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
908 intr_flag, sc->irq_rid);
912 err = bus_setup_intr(sc->dev, sc->irq_res,
913 INTR_MPSAFE | INTR_TYPE_NET,
917 sc->cxgb_intr, sc, &sc->intr_tag);
920 device_printf(sc->dev,
921 "Cannot set up interrupt (%x, %u, %d)\n",
922 intr_flag, sc->irq_rid, err);
923 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
925 sc->irq_res = sc->intr_tag = NULL;
930 /* That's all for INTx or MSI */
931 if (!(intr_flag & USING_MSIX) || err)
934 for (i = 0; i < sc->msi_count - 1; i++) {
936 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
937 RF_SHAREABLE | RF_ACTIVE);
939 device_printf(sc->dev, "Cannot allocate interrupt "
940 "for message %d\n", rid);
945 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
949 t3_intr_msix, &sc->sge.qs[i], &tag);
951 device_printf(sc->dev, "Cannot set up interrupt "
952 "for message %d (%d)\n", rid, err);
953 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
957 sc->msix_irq_rid[i] = rid;
958 sc->msix_irq_res[i] = res;
959 sc->msix_intr_tag[i] = tag;
963 cxgb_teardown_interrupts(sc);
970 cxgb_port_probe(device_t dev)
976 p = device_get_softc(dev);
978 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
979 device_set_desc_copy(dev, buf);
985 cxgb_makedev(struct port_info *pi)
988 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
989 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
991 if (pi->port_cdev == NULL)
994 pi->port_cdev->si_drv1 = (void *)pi;
999 #ifndef LRO_SUPPORTED
1003 #define IFCAP_LRO 0x0
1006 #ifdef TSO_SUPPORTED
1007 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO)
1008 /* Don't enable TSO6 yet */
1009 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU | IFCAP_LRO)
1011 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
1012 /* Don't enable TSO6 yet */
1013 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
1014 #define IFCAP_TSO4 0x0
1015 #define IFCAP_TSO6 0x0
1016 #define CSUM_TSO 0x0
1021 cxgb_port_attach(device_t dev)
1023 struct port_info *p;
1029 p = device_get_softc(dev);
1031 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1032 device_get_unit(device_get_parent(dev)), p->port_id);
1033 PORT_LOCK_INIT(p, p->lockbuf);
1035 /* Allocate an ifnet object and set it up */
1036 ifp = p->ifp = if_alloc(IFT_ETHER);
1038 device_printf(dev, "Cannot allocate ifnet\n");
1043 * Note that there is currently no watchdog timer.
1045 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1046 ifp->if_init = cxgb_init;
1048 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1049 ifp->if_ioctl = cxgb_ioctl;
1050 ifp->if_start = cxgb_start;
1053 ifp->if_timer = 0; /* Disable ifnet watchdog */
1054 ifp->if_watchdog = NULL;
1056 ifp->if_snd.ifq_drv_maxlen = cxgb_snd_queue_len;
1057 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
1058 IFQ_SET_READY(&ifp->if_snd);
1060 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
1061 ifp->if_capabilities |= CXGB_CAP;
1062 ifp->if_capenable |= CXGB_CAP_ENABLE;
1063 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
1065 * disable TSO on 4-port - it isn't supported by the firmware yet
1067 if (p->adapter->params.nports > 2) {
1068 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6);
1069 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6);
1070 ifp->if_hwassist &= ~CSUM_TSO;
1073 ether_ifattach(ifp, p->hw_addr);
1074 ifp->if_transmit = cxgb_transmit;
1075 ifp->if_qflush = cxgb_qflush;
1078 * Only default to jumbo frames on 10GigE
1080 if (p->adapter->params.nports <= 2)
1081 ifp->if_mtu = ETHERMTU_JUMBO;
1082 if ((err = cxgb_makedev(p)) != 0) {
1083 printf("makedev failed %d\n", err);
1087 /* Create a list of media supported by this port */
1088 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1090 cxgb_build_medialist(p);
1092 t3_sge_init_port(p);
1098 * cxgb_port_detach() is called via the device_detach methods when
1099 * cxgb_free() calls the bus_generic_detach. It is responsible for
1100 * removing the device from the view of the kernel, i.e. from all
1101 * interfaces lists etc. This routine is only called when the driver is
1102 * being unloaded, not when the link goes down.
1105 cxgb_port_detach(device_t dev)
1107 struct port_info *p;
1111 p = device_get_softc(dev);
1114 cxgb_begin_detach(p);
1116 if (p->port_cdev != NULL)
1117 destroy_dev(p->port_cdev);
1119 cxgb_uninit_synchronized(p);
1120 ether_ifdetach(p->ifp);
1122 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1123 struct sge_qset *qs = &sc->sge.qs[i];
1124 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1126 callout_drain(&txq->txq_watchdog);
1127 callout_drain(&txq->txq_timer);
1130 PORT_LOCK_DEINIT(p);
1139 t3_fatal_err(struct adapter *sc)
1143 if (sc->flags & FULL_INIT_DONE) {
1145 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1146 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1147 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1148 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1149 t3_intr_disable(sc);
1151 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1152 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1153 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1154 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1158 t3_os_find_pci_capability(adapter_t *sc, int cap)
1161 struct pci_devinfo *dinfo;
1167 dinfo = device_get_ivars(dev);
1170 status = pci_read_config(dev, PCIR_STATUS, 2);
1171 if (!(status & PCIM_STATUS_CAPPRESENT))
1174 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1180 ptr = PCIR_CAP_PTR_2;
1186 ptr = pci_read_config(dev, ptr, 1);
1189 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1191 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1198 t3_os_pci_save_state(struct adapter *sc)
1201 struct pci_devinfo *dinfo;
1204 dinfo = device_get_ivars(dev);
1206 pci_cfg_save(dev, dinfo, 0);
1211 t3_os_pci_restore_state(struct adapter *sc)
1214 struct pci_devinfo *dinfo;
1217 dinfo = device_get_ivars(dev);
1219 pci_cfg_restore(dev, dinfo);
1224 * t3_os_link_changed - handle link status changes
1225 * @adapter: the adapter associated with the link change
1226 * @port_id: the port index whose limk status has changed
1227 * @link_status: the new status of the link
1228 * @speed: the new speed setting
1229 * @duplex: the new duplex setting
1230 * @fc: the new flow-control setting
1232 * This is the OS-dependent handler for link status changes. The OS
1233 * neutral handler takes care of most of the processing for these events,
1234 * then calls this handler for any OS-specific processing.
1237 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1240 struct port_info *pi = &adapter->port[port_id];
1241 struct ifnet *ifp = pi->ifp;
1243 /* no race with detach, so ifp should always be good */
1244 KASSERT(ifp, ("%s: if detached.", __func__));
1247 ifp->if_baudrate = IF_Mbps(speed);
1248 if_link_state_change(ifp, LINK_STATE_UP);
1250 if_link_state_change(ifp, LINK_STATE_DOWN);
1254 * t3_os_phymod_changed - handle PHY module changes
1255 * @phy: the PHY reporting the module change
1256 * @mod_type: new module type
1258 * This is the OS-dependent handler for PHY module changes. It is
1259 * invoked when a PHY module is removed or inserted for any OS-specific
1262 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1264 static const char *mod_str[] = {
1265 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
1267 struct port_info *pi = &adap->port[port_id];
1268 int mod = pi->phy.modtype;
1270 if (mod != pi->media.ifm_cur->ifm_data)
1271 cxgb_build_medialist(pi);
1273 if (mod == phy_modtype_none)
1274 if_printf(pi->ifp, "PHY module unplugged\n");
1276 KASSERT(mod < ARRAY_SIZE(mod_str),
1277 ("invalid PHY module type %d", mod));
1278 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1283 * Interrupt-context handler for external (PHY) interrupts.
1286 t3_os_ext_intr_handler(adapter_t *sc)
1289 printf("t3_os_ext_intr_handler\n");
1291 * Schedule a task to handle external interrupts as they may be slow
1292 * and we use a mutex to protect MDIO registers. We disable PHY
1293 * interrupts in the meantime and let the task reenable them when
1296 if (sc->slow_intr_mask) {
1298 sc->slow_intr_mask &= ~F_T3DBG;
1299 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1300 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1306 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1310 * The ifnet might not be allocated before this gets called,
1311 * as this is called early on in attach by t3_prep_adapter
1312 * save the address off in the port structure
1315 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1316 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1320 * Programs the XGMAC based on the settings in the ifnet. These settings
1321 * include MTU, MAC address, mcast addresses, etc.
1324 cxgb_update_mac_settings(struct port_info *p)
1326 struct ifnet *ifp = p->ifp;
1327 struct t3_rx_mode rm;
1328 struct cmac *mac = &p->mac;
1331 PORT_LOCK_ASSERT_OWNED(p);
1333 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1336 if (ifp->if_capenable & IFCAP_VLAN_MTU)
1337 mtu += ETHER_VLAN_ENCAP_LEN;
1339 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1341 t3_mac_set_mtu(mac, mtu);
1342 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1343 t3_mac_set_address(mac, 0, p->hw_addr);
1344 t3_init_rx_mode(&rm, p);
1345 t3_mac_set_rx_mode(mac, &rm);
1350 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1355 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1364 init_tp_parity(struct adapter *adap)
1368 struct cpl_set_tcb_field *greq;
1369 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1371 t3_tp_set_offload_mode(adap, 1);
1373 for (i = 0; i < 16; i++) {
1374 struct cpl_smt_write_req *req;
1376 m = m_gethdr(M_WAITOK, MT_DATA);
1377 req = mtod(m, struct cpl_smt_write_req *);
1378 m->m_len = m->m_pkthdr.len = sizeof(*req);
1379 memset(req, 0, sizeof(*req));
1380 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1381 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1383 t3_mgmt_tx(adap, m);
1386 for (i = 0; i < 2048; i++) {
1387 struct cpl_l2t_write_req *req;
1389 m = m_gethdr(M_WAITOK, MT_DATA);
1390 req = mtod(m, struct cpl_l2t_write_req *);
1391 m->m_len = m->m_pkthdr.len = sizeof(*req);
1392 memset(req, 0, sizeof(*req));
1393 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1394 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1395 req->params = htonl(V_L2T_W_IDX(i));
1396 t3_mgmt_tx(adap, m);
1399 for (i = 0; i < 2048; i++) {
1400 struct cpl_rte_write_req *req;
1402 m = m_gethdr(M_WAITOK, MT_DATA);
1403 req = mtod(m, struct cpl_rte_write_req *);
1404 m->m_len = m->m_pkthdr.len = sizeof(*req);
1405 memset(req, 0, sizeof(*req));
1406 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1407 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1408 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1409 t3_mgmt_tx(adap, m);
1412 m = m_gethdr(M_WAITOK, MT_DATA);
1413 greq = mtod(m, struct cpl_set_tcb_field *);
1414 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1415 memset(greq, 0, sizeof(*greq));
1416 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1417 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1418 greq->mask = htobe64(1);
1419 t3_mgmt_tx(adap, m);
1421 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1422 t3_tp_set_offload_mode(adap, 0);
1427 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1428 * @adap: the adapter
1430 * Sets up RSS to distribute packets to multiple receive queues. We
1431 * configure the RSS CPU lookup table to distribute to the number of HW
1432 * receive queues, and the response queue lookup table to narrow that
1433 * down to the response queues actually configured for each port.
1434 * We always configure the RSS mapping for two ports since the mapping
1435 * table has plenty of entries.
1438 setup_rss(adapter_t *adap)
1442 uint8_t cpus[SGE_QSETS + 1];
1443 uint16_t rspq_map[RSS_TABLE_SIZE];
1445 for (i = 0; i < SGE_QSETS; ++i)
1447 cpus[SGE_QSETS] = 0xff;
1450 for_each_port(adap, i) {
1451 const struct port_info *pi = adap2pinfo(adap, i);
1453 nq[pi->tx_chan] += pi->nqsets;
1455 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1456 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1457 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1459 /* Calculate the reverse RSS map table */
1460 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1461 if (adap->rrss_map[rspq_map[i]] == 0xff)
1462 adap->rrss_map[rspq_map[i]] = i;
1464 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1465 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1466 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1472 * Sends an mbuf to an offload queue driver
1473 * after dealing with any active network taps.
1476 offload_tx(struct t3cdev *tdev, struct mbuf *m)
1480 ret = t3_offload_tx(tdev, m);
1485 write_smt_entry(struct adapter *adapter, int idx)
1487 struct port_info *pi = &adapter->port[idx];
1488 struct cpl_smt_write_req *req;
1491 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1494 req = mtod(m, struct cpl_smt_write_req *);
1495 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
1497 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1498 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1499 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1501 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1502 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1504 m_set_priority(m, 1);
1506 offload_tx(&adapter->tdev, m);
1512 init_smt(struct adapter *adapter)
1516 for_each_port(adapter, i)
1517 write_smt_entry(adapter, i);
1522 init_port_mtus(adapter_t *adapter)
1524 unsigned int mtus = ETHERMTU | (ETHERMTU << 16);
1526 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1530 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1534 struct mngt_pktsched_wr *req;
1536 m = m_gethdr(M_DONTWAIT, MT_DATA);
1538 req = mtod(m, struct mngt_pktsched_wr *);
1539 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1540 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1545 req->binding = port;
1546 m->m_len = m->m_pkthdr.len = sizeof(*req);
1547 t3_mgmt_tx(adap, m);
1552 bind_qsets(adapter_t *sc)
1556 for (i = 0; i < (sc)->params.nports; ++i) {
1557 const struct port_info *pi = adap2pinfo(sc, i);
1559 for (j = 0; j < pi->nqsets; ++j) {
1560 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1568 update_tpeeprom(struct adapter *adap)
1570 #ifdef FIRMWARE_LATEST
1571 const struct firmware *tpeeprom;
1573 struct firmware *tpeeprom;
1577 unsigned int major, minor;
1581 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1583 major = G_TP_VERSION_MAJOR(version);
1584 minor = G_TP_VERSION_MINOR(version);
1585 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1588 rev = t3rev2char(adap);
1589 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1591 tpeeprom = firmware_get(name);
1592 if (tpeeprom == NULL) {
1593 device_printf(adap->dev,
1594 "could not load TP EEPROM: unable to load %s\n",
1599 len = tpeeprom->datasize - 4;
1601 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1603 goto release_tpeeprom;
1605 if (len != TP_SRAM_LEN) {
1606 device_printf(adap->dev,
1607 "%s length is wrong len=%d expected=%d\n", name,
1612 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1616 device_printf(adap->dev,
1617 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1618 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1620 device_printf(adap->dev,
1621 "Protocol SRAM image update in EEPROM failed\n");
1624 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1630 update_tpsram(struct adapter *adap)
1632 #ifdef FIRMWARE_LATEST
1633 const struct firmware *tpsram;
1635 struct firmware *tpsram;
1640 rev = t3rev2char(adap);
1641 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1643 update_tpeeprom(adap);
1645 tpsram = firmware_get(name);
1646 if (tpsram == NULL){
1647 device_printf(adap->dev, "could not load TP SRAM\n");
1650 device_printf(adap->dev, "updating TP SRAM\n");
1652 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1654 goto release_tpsram;
1656 ret = t3_set_proto_sram(adap, tpsram->data);
1658 device_printf(adap->dev, "loading protocol SRAM failed\n");
1661 firmware_put(tpsram, FIRMWARE_UNLOAD);
1667 * cxgb_up - enable the adapter
1668 * @adap: adapter being enabled
1670 * Called when the first port is enabled, this function performs the
1671 * actions necessary to make an adapter operational, such as completing
1672 * the initialization of HW modules, and enabling interrupts.
1675 cxgb_up(struct adapter *sc)
1679 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1680 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1681 __func__, sc->open_device_map));
1683 if ((sc->flags & FULL_INIT_DONE) == 0) {
1685 if ((sc->flags & FW_UPTODATE) == 0)
1686 if ((err = upgrade_fw(sc)))
1689 if ((sc->flags & TPS_UPTODATE) == 0)
1690 if ((err = update_tpsram(sc)))
1693 err = t3_init_hw(sc, 0);
1697 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1698 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1700 err = setup_sge_qsets(sc);
1707 err = cxgb_setup_interrupts(sc);
1711 t3_add_configured_sysctls(sc);
1712 sc->flags |= FULL_INIT_DONE;
1719 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1720 is_offload(sc) && init_tp_parity(sc) == 0)
1721 sc->flags |= TP_PARITY_INIT;
1723 if (sc->flags & TP_PARITY_INIT) {
1724 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1725 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1728 if (!(sc->flags & QUEUES_BOUND)) {
1730 sc->flags |= QUEUES_BOUND;
1733 t3_sge_reset_adapter(sc);
1739 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1740 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1741 * during controller_detach, not here.
1744 cxgb_down(struct adapter *sc)
1746 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1749 t3_intr_disable(sc);
1753 offload_open(struct port_info *pi)
1755 struct adapter *sc = pi->adapter;
1756 struct t3cdev *tdev = &sc->tdev;
1758 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1760 setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT);
1762 t3_tp_set_offload_mode(sc, 1);
1763 tdev->lldev = pi->ifp;
1765 t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
1766 sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff);
1768 cxgb_add_clients(tdev);
1774 offload_close(struct t3cdev *tdev)
1776 struct adapter *adapter = tdev2adap(tdev);
1778 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1781 /* Call back all registered clients */
1782 cxgb_remove_clients(tdev);
1785 cxgb_set_dummy_ops(tdev);
1786 t3_tp_set_offload_mode(adapter, 0);
1788 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1794 * Begin a synchronized operation. If this call succeeds, it is guaranteed that
1795 * no one will remove the port or its ifp from underneath the caller. Caller is
1796 * also granted exclusive access to open_device_map.
1798 * operation here means init, uninit, detach, and ioctl service.
1801 * EINTR (ctrl-c pressed during ifconfig for example).
1802 * ENXIO (port is about to detach - due to kldunload for example).
1805 cxgb_begin_op(struct port_info *p, const char *wmsg)
1808 struct adapter *sc = p->adapter;
1812 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1813 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, wmsg, 0)) {
1821 else if (!IS_BUSY(sc))
1824 KASSERT(0, ("%s: port %d, p->flags = %x , sc->flags = %x",
1825 __func__, p->port_id, p->flags, sc->flags));
1835 * End a synchronized operation. Read comment block above cxgb_begin_op.
1838 cxgb_end_op(struct port_info *p)
1840 struct adapter *sc = p->adapter;
1843 KASSERT(IS_BUSY(sc), ("%s: not busy.", __func__));
1845 wakeup_one(&sc->flags);
1852 * Prepare for port detachment. Detach is a special kind of synchronized
1853 * operation. Also read comment before cxgb_begin_op.
1856 cxgb_begin_detach(struct port_info *p)
1858 struct adapter *sc = p->adapter;
1861 * Inform those waiting for this port that it is going to be destroyed
1862 * and they should not continue further. (They'll return with ENXIO).
1870 * Wait for in-progress operations.
1873 while (IS_BUSY(sc)) {
1874 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1883 * if_init for cxgb ports.
1886 cxgb_init(void *arg)
1888 struct port_info *p = arg;
1890 if (cxgb_begin_op(p, "cxgbinit"))
1893 cxgb_init_synchronized(p);
1898 cxgb_init_synchronized(struct port_info *p)
1900 struct adapter *sc = p->adapter;
1901 struct ifnet *ifp = p->ifp;
1902 struct cmac *mac = &p->mac;
1905 if (sc->open_device_map == 0) {
1906 if ((rc = cxgb_up(sc)) != 0)
1909 if (is_offload(sc) && !ofld_disable && offload_open(p))
1911 "Could not initialize offload capabilities\n");
1915 t3_port_intr_enable(sc, p->port_id);
1916 if (!mac->multiport)
1918 cxgb_update_mac_settings(p);
1919 t3_link_start(&p->phy, mac, &p->link_config);
1920 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1921 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1922 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1925 t3_link_changed(sc, p->port_id);
1927 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1928 struct sge_qset *qs = &sc->sge.qs[i];
1929 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1931 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1932 txq->txq_watchdog.c_cpu);
1936 setbit(&sc->open_device_map, p->port_id);
1942 * Called on "ifconfig down", and from port_detach
1945 cxgb_uninit_synchronized(struct port_info *pi)
1947 struct adapter *sc = pi->adapter;
1948 struct ifnet *ifp = pi->ifp;
1951 * Clear this port's bit from the open device map, and then drain all
1952 * the tasks that can access/manipulate this port's port_info or ifp.
1953 * We disable this port's interrupts here and so the the slow/ext
1954 * interrupt tasks won't be enqueued. The tick task will continue to
1955 * be enqueued every second but the runs after this drain will not see
1956 * this port in the open device map.
1958 * A well behaved task must take open_device_map into account and ignore
1959 * ports that are not open.
1961 clrbit(&sc->open_device_map, pi->port_id);
1962 t3_port_intr_disable(sc, pi->port_id);
1963 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1964 taskqueue_drain(sc->tq, &sc->ext_intr_task);
1965 taskqueue_drain(sc->tq, &sc->tick_task);
1968 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1970 /* disable pause frames */
1971 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1973 /* Reset RX FIFO HWM */
1974 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1975 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1979 /* Wait for TXFIFO empty */
1980 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1981 F_TXFIFO_EMPTY, 1, 20, 5);
1984 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1987 pi->phy.ops->power_down(&pi->phy, 1);
1991 pi->link_config.link_ok = 0;
1992 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0);
1994 if ((sc->open_device_map & PORT_MASK) == 0)
1995 offload_close(&sc->tdev);
1997 if (sc->open_device_map == 0)
1998 cxgb_down(pi->adapter);
2003 #ifdef LRO_SUPPORTED
2005 * Mark lro enabled or disabled in all qsets for this port
2008 cxgb_set_lro(struct port_info *p, int enabled)
2011 struct adapter *adp = p->adapter;
2014 PORT_LOCK_ASSERT_OWNED(p);
2015 for (i = 0; i < p->nqsets; i++) {
2016 q = &adp->sge.qs[p->first_qset + i];
2017 q->lro.enabled = (enabled != 0);
2024 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
2026 struct port_info *p = ifp->if_softc;
2027 struct ifreq *ifr = (struct ifreq *)data;
2028 int flags, error = 0, mtu, handle_unsynchronized = 0;
2031 if ((error = cxgb_begin_op(p, "cxgbioct")) != 0)
2035 * Only commands that should be handled within begin-op/end-op are
2036 * serviced in this switch statement. See handle_unsynchronized.
2041 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
2046 cxgb_update_mac_settings(p);
2052 if (ifp->if_flags & IFF_UP) {
2053 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2054 flags = p->if_flags;
2055 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
2056 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
2058 cxgb_update_mac_settings(p);
2062 error = cxgb_init_synchronized(p);
2063 p->if_flags = ifp->if_flags;
2064 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2065 error = cxgb_uninit_synchronized(p);
2070 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2072 cxgb_update_mac_settings(p);
2078 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2079 if (mask & IFCAP_TXCSUM) {
2080 if (IFCAP_TXCSUM & ifp->if_capenable) {
2081 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
2082 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
2083 | CSUM_IP | CSUM_TSO);
2085 ifp->if_capenable |= IFCAP_TXCSUM;
2086 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP
2090 if (mask & IFCAP_RXCSUM) {
2091 ifp->if_capenable ^= IFCAP_RXCSUM;
2093 if (mask & IFCAP_TSO4) {
2094 if (IFCAP_TSO4 & ifp->if_capenable) {
2095 ifp->if_capenable &= ~IFCAP_TSO4;
2096 ifp->if_hwassist &= ~CSUM_TSO;
2097 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
2098 ifp->if_capenable |= IFCAP_TSO4;
2099 ifp->if_hwassist |= CSUM_TSO;
2103 #ifdef LRO_SUPPORTED
2104 if (mask & IFCAP_LRO) {
2105 ifp->if_capenable ^= IFCAP_LRO;
2107 /* Safe to do this even if cxgb_up not called yet */
2108 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2111 if (mask & IFCAP_VLAN_HWTAGGING) {
2112 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2113 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2115 cxgb_update_mac_settings(p);
2119 if (mask & IFCAP_VLAN_MTU) {
2120 ifp->if_capenable ^= IFCAP_VLAN_MTU;
2121 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2123 cxgb_update_mac_settings(p);
2127 if (mask & IFCAP_VLAN_HWCSUM) {
2128 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2131 #ifdef VLAN_CAPABILITIES
2132 VLAN_CAPABILITIES(ifp);
2136 handle_unsynchronized = 1;
2141 * We don't want to call anything outside the driver while inside a
2142 * begin-op/end-op block. If it calls us back (eg. ether_ioctl may
2143 * call cxgb_init) we may deadlock if the state is already marked busy.
2145 * XXX: this probably opens a small race window with kldunload...
2149 /* The IS_DOOMED check is racy, we're clutching at straws here */
2150 if (handle_unsynchronized && !IS_DOOMED(p)) {
2151 if (command == SIOCSIFMEDIA || command == SIOCGIFMEDIA)
2152 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2154 error = ether_ioctl(ifp, command, data);
2161 cxgb_media_change(struct ifnet *ifp)
2163 return (EOPNOTSUPP);
2167 * Translates phy->modtype to the correct Ethernet media subtype.
2170 cxgb_ifm_type(int mod)
2173 case phy_modtype_sr:
2174 return (IFM_10G_SR);
2175 case phy_modtype_lr:
2176 return (IFM_10G_LR);
2177 case phy_modtype_lrm:
2178 return (IFM_10G_LRM);
2179 case phy_modtype_twinax:
2180 return (IFM_10G_TWINAX);
2181 case phy_modtype_twinax_long:
2182 return (IFM_10G_TWINAX_LONG);
2183 case phy_modtype_none:
2185 case phy_modtype_unknown:
2186 return (IFM_UNKNOWN);
2189 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2190 return (IFM_UNKNOWN);
2194 * Rebuilds the ifmedia list for this port, and sets the current media.
2197 cxgb_build_medialist(struct port_info *p)
2199 struct cphy *phy = &p->phy;
2200 struct ifmedia *media = &p->media;
2201 int mod = phy->modtype;
2202 int m = IFM_ETHER | IFM_FDX;
2206 ifmedia_removeall(media);
2207 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2210 if (phy->caps & SUPPORTED_10000baseT_Full)
2211 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2213 if (phy->caps & SUPPORTED_1000baseT_Full)
2214 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2216 if (phy->caps & SUPPORTED_100baseT_Full)
2217 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2219 if (phy->caps & SUPPORTED_10baseT_Full)
2220 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2222 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2223 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2225 } else if (phy->caps & SUPPORTED_TP) {
2228 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2229 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2231 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2232 ifmedia_set(media, m | IFM_10G_CX4);
2234 } else if (phy->caps & SUPPORTED_FIBRE &&
2235 phy->caps & SUPPORTED_10000baseT_Full) {
2236 /* 10G optical (but includes SFP+ twinax) */
2238 m |= cxgb_ifm_type(mod);
2239 if (IFM_SUBTYPE(m) == IFM_NONE)
2242 ifmedia_add(media, m, mod, NULL);
2243 ifmedia_set(media, m);
2245 } else if (phy->caps & SUPPORTED_FIBRE &&
2246 phy->caps & SUPPORTED_1000baseT_Full) {
2249 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2250 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2251 ifmedia_set(media, m | IFM_1000_SX);
2254 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2262 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2264 struct port_info *p = ifp->if_softc;
2265 struct ifmedia_entry *cur = p->media.ifm_cur;
2266 int speed = p->link_config.speed;
2268 if (cur->ifm_data != p->phy.modtype) {
2269 cxgb_build_medialist(p);
2270 cur = p->media.ifm_cur;
2273 ifmr->ifm_status = IFM_AVALID;
2274 if (!p->link_config.link_ok)
2277 ifmr->ifm_status |= IFM_ACTIVE;
2280 * active and current will differ iff current media is autoselect. That
2281 * can happen only for copper RJ45.
2283 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2285 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2286 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2288 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2289 if (speed == SPEED_10000)
2290 ifmr->ifm_active |= IFM_10G_T;
2291 else if (speed == SPEED_1000)
2292 ifmr->ifm_active |= IFM_1000_T;
2293 else if (speed == SPEED_100)
2294 ifmr->ifm_active |= IFM_100_TX;
2295 else if (speed == SPEED_10)
2296 ifmr->ifm_active |= IFM_10_T;
2298 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2303 cxgb_async_intr(void *data)
2305 adapter_t *sc = data;
2308 device_printf(sc->dev, "cxgb_async_intr\n");
2310 * May need to sleep - defer to taskqueue
2312 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2316 cxgb_ext_intr_handler(void *arg, int count)
2318 adapter_t *sc = (adapter_t *)arg;
2321 printf("cxgb_ext_intr_handler\n");
2323 t3_phy_intr_handler(sc);
2325 /* Now reenable external interrupts */
2327 if (sc->slow_intr_mask) {
2328 sc->slow_intr_mask |= F_T3DBG;
2329 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
2330 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
2336 check_link_status(adapter_t *sc)
2340 for (i = 0; i < (sc)->params.nports; ++i) {
2341 struct port_info *p = &sc->port[i];
2343 if (!isset(&sc->open_device_map, p->port_id))
2346 if (p->link_fault || !(p->phy.caps & SUPPORTED_IRQ))
2347 t3_link_changed(sc, i);
2352 check_t3b2_mac(struct adapter *sc)
2356 if (sc->flags & CXGB_SHUTDOWN)
2359 for_each_port(sc, i) {
2360 struct port_info *p = &sc->port[i];
2363 struct ifnet *ifp = p->ifp;
2366 if (!isset(&sc->open_device_map, p->port_id))
2369 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2370 ("%s: state mismatch (drv_flags %x, device_map %x)",
2371 __func__, ifp->if_drv_flags, sc->open_device_map));
2374 status = t3b2_mac_watchdog_task(&p->mac);
2376 p->mac.stats.num_toggled++;
2377 else if (status == 2) {
2378 struct cmac *mac = &p->mac;
2380 cxgb_update_mac_settings(p);
2381 t3_link_start(&p->phy, mac, &p->link_config);
2382 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2383 t3_port_intr_enable(sc, p->port_id);
2384 p->mac.stats.num_resets++;
2391 cxgb_tick(void *arg)
2393 adapter_t *sc = (adapter_t *)arg;
2395 if (sc->flags & CXGB_SHUTDOWN)
2398 taskqueue_enqueue(sc->tq, &sc->tick_task);
2399 callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
2403 cxgb_tick_handler(void *arg, int count)
2405 adapter_t *sc = (adapter_t *)arg;
2406 const struct adapter_params *p = &sc->params;
2408 uint32_t cause, reset;
2410 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2413 check_link_status(sc);
2414 sc->check_task_cnt++;
2416 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2419 cause = t3_read_reg(sc, A_SG_INT_CAUSE);
2421 if (cause & F_FLEMPTY) {
2422 struct sge_qset *qs = &sc->sge.qs[0];
2427 cause = (t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) >>
2428 S_FL0EMPTY) & 0xffff;
2430 qs->fl[i].empty += (cause & 1);
2437 t3_write_reg(sc, A_SG_INT_CAUSE, reset);
2439 for (i = 0; i < sc->params.nports; i++) {
2440 struct port_info *pi = &sc->port[i];
2441 struct ifnet *ifp = pi->ifp;
2442 struct cmac *mac = &pi->mac;
2443 struct mac_stats *mstats = &mac->stats;
2445 if (!isset(&sc->open_device_map, pi->port_id))
2449 t3_mac_update_stats(mac);
2453 mstats->tx_frames_64 +
2454 mstats->tx_frames_65_127 +
2455 mstats->tx_frames_128_255 +
2456 mstats->tx_frames_256_511 +
2457 mstats->tx_frames_512_1023 +
2458 mstats->tx_frames_1024_1518 +
2459 mstats->tx_frames_1519_max;
2462 mstats->rx_frames_64 +
2463 mstats->rx_frames_65_127 +
2464 mstats->rx_frames_128_255 +
2465 mstats->rx_frames_256_511 +
2466 mstats->rx_frames_512_1023 +
2467 mstats->rx_frames_1024_1518 +
2468 mstats->rx_frames_1519_max;
2470 ifp->if_obytes = mstats->tx_octets;
2471 ifp->if_ibytes = mstats->rx_octets;
2472 ifp->if_omcasts = mstats->tx_mcast_frames;
2473 ifp->if_imcasts = mstats->rx_mcast_frames;
2475 ifp->if_collisions =
2476 mstats->tx_total_collisions;
2478 ifp->if_iqdrops = mstats->rx_cong_drops;
2481 mstats->tx_excess_collisions +
2482 mstats->tx_underrun +
2483 mstats->tx_len_errs +
2484 mstats->tx_mac_internal_errs +
2485 mstats->tx_excess_deferral +
2486 mstats->tx_fcs_errs;
2489 mstats->rx_data_errs +
2490 mstats->rx_sequence_errs +
2492 mstats->rx_too_long +
2493 mstats->rx_mac_internal_errs +
2495 mstats->rx_fcs_errs;
2500 /* Count rx fifo overflows, once per second */
2501 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2503 if (cause & F_RXFIFO_OVERFLOW) {
2504 mac->stats.rx_fifo_ovfl++;
2505 reset |= F_RXFIFO_OVERFLOW;
2507 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2512 touch_bars(device_t dev)
2517 #if !defined(__LP64__) && 0
2520 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2521 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2522 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2523 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2524 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2525 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2530 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2534 u32 aligned_offset, aligned_len, *p;
2535 struct adapter *adapter = pi->adapter;
2538 aligned_offset = offset & ~3;
2539 aligned_len = (len + (offset & 3) + 3) & ~3;
2541 if (aligned_offset != offset || aligned_len != len) {
2542 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2545 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2546 if (!err && aligned_len > 4)
2547 err = t3_seeprom_read(adapter,
2548 aligned_offset + aligned_len - 4,
2549 (u32 *)&buf[aligned_len - 4]);
2552 memcpy(buf + (offset & 3), data, len);
2554 buf = (uint8_t *)(uintptr_t)data;
2556 err = t3_seeprom_wp(adapter, 0);
2560 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2561 err = t3_seeprom_write(adapter, aligned_offset, *p);
2562 aligned_offset += 4;
2566 err = t3_seeprom_wp(adapter, 1);
2569 free(buf, M_DEVBUF);
2575 in_range(int val, int lo, int hi)
2577 return val < 0 || (val <= hi && val >= lo);
2581 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2587 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2593 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2594 int fflag, struct thread *td)
2597 struct port_info *pi = dev->si_drv1;
2598 adapter_t *sc = pi->adapter;
2600 #ifdef PRIV_SUPPORTED
2601 if (priv_check(td, PRIV_DRIVER)) {
2603 printf("user does not have access to privileged ioctls\n");
2609 printf("user does not have access to privileged ioctls\n");
2615 case CHELSIO_GET_MIIREG: {
2617 struct cphy *phy = &pi->phy;
2618 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2620 if (!phy->mdio_read)
2621 return (EOPNOTSUPP);
2623 mmd = mid->phy_id >> 8;
2626 else if (mmd > MDIO_DEV_VEND2)
2629 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2630 mid->reg_num, &val);
2632 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2633 mid->reg_num & 0x1f, &val);
2638 case CHELSIO_SET_MIIREG: {
2639 struct cphy *phy = &pi->phy;
2640 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2642 if (!phy->mdio_write)
2643 return (EOPNOTSUPP);
2645 mmd = mid->phy_id >> 8;
2648 else if (mmd > MDIO_DEV_VEND2)
2651 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2652 mmd, mid->reg_num, mid->val_in);
2654 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2655 mid->reg_num & 0x1f,
2659 case CHELSIO_SETREG: {
2660 struct ch_reg *edata = (struct ch_reg *)data;
2661 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2663 t3_write_reg(sc, edata->addr, edata->val);
2666 case CHELSIO_GETREG: {
2667 struct ch_reg *edata = (struct ch_reg *)data;
2668 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2670 edata->val = t3_read_reg(sc, edata->addr);
2673 case CHELSIO_GET_SGE_CONTEXT: {
2674 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2675 mtx_lock_spin(&sc->sge.reg_lock);
2676 switch (ecntxt->cntxt_type) {
2677 case CNTXT_TYPE_EGRESS:
2678 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2682 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2685 case CNTXT_TYPE_RSP:
2686 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2690 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2697 mtx_unlock_spin(&sc->sge.reg_lock);
2700 case CHELSIO_GET_SGE_DESC: {
2701 struct ch_desc *edesc = (struct ch_desc *)data;
2703 if (edesc->queue_num >= SGE_QSETS * 6)
2705 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2706 edesc->queue_num % 6, edesc->idx, edesc->data);
2712 case CHELSIO_GET_QSET_PARAMS: {
2713 struct qset_params *q;
2714 struct ch_qset_params *t = (struct ch_qset_params *)data;
2715 int q1 = pi->first_qset;
2716 int nqsets = pi->nqsets;
2719 if (t->qset_idx >= nqsets)
2722 i = q1 + t->qset_idx;
2723 q = &sc->params.sge.qset[i];
2724 t->rspq_size = q->rspq_size;
2725 t->txq_size[0] = q->txq_size[0];
2726 t->txq_size[1] = q->txq_size[1];
2727 t->txq_size[2] = q->txq_size[2];
2728 t->fl_size[0] = q->fl_size;
2729 t->fl_size[1] = q->jumbo_size;
2730 t->polling = q->polling;
2732 t->intr_lat = q->coalesce_usecs;
2733 t->cong_thres = q->cong_thres;
2736 if (sc->flags & USING_MSIX)
2737 t->vector = rman_get_start(sc->msix_irq_res[i]);
2739 t->vector = rman_get_start(sc->irq_res);
2743 case CHELSIO_GET_QSET_NUM: {
2744 struct ch_reg *edata = (struct ch_reg *)data;
2745 edata->val = pi->nqsets;
2748 case CHELSIO_LOAD_FW: {
2751 struct ch_mem_range *t = (struct ch_mem_range *)data;
2754 * You're allowed to load a firmware only before FULL_INIT_DONE
2756 * FW_UPTODATE is also set so the rest of the initialization
2757 * will not overwrite what was loaded here. This gives you the
2758 * flexibility to load any firmware (and maybe shoot yourself in
2763 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2768 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2772 error = copyin(t->buf, fw_data, t->len);
2775 error = -t3_load_fw(sc, fw_data, t->len);
2777 if (t3_get_fw_version(sc, &vers) == 0) {
2778 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2779 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2780 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2784 sc->flags |= FW_UPTODATE;
2786 free(fw_data, M_DEVBUF);
2790 case CHELSIO_LOAD_BOOT: {
2792 struct ch_mem_range *t = (struct ch_mem_range *)data;
2794 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2798 error = copyin(t->buf, boot_data, t->len);
2800 error = -t3_load_boot(sc, boot_data, t->len);
2802 free(boot_data, M_DEVBUF);
2805 case CHELSIO_GET_PM: {
2806 struct ch_pm *m = (struct ch_pm *)data;
2807 struct tp_params *p = &sc->params.tp;
2809 if (!is_offload(sc))
2810 return (EOPNOTSUPP);
2812 m->tx_pg_sz = p->tx_pg_size;
2813 m->tx_num_pg = p->tx_num_pgs;
2814 m->rx_pg_sz = p->rx_pg_size;
2815 m->rx_num_pg = p->rx_num_pgs;
2816 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2820 case CHELSIO_SET_PM: {
2821 struct ch_pm *m = (struct ch_pm *)data;
2822 struct tp_params *p = &sc->params.tp;
2824 if (!is_offload(sc))
2825 return (EOPNOTSUPP);
2826 if (sc->flags & FULL_INIT_DONE)
2829 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2830 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2831 return (EINVAL); /* not power of 2 */
2832 if (!(m->rx_pg_sz & 0x14000))
2833 return (EINVAL); /* not 16KB or 64KB */
2834 if (!(m->tx_pg_sz & 0x1554000))
2836 if (m->tx_num_pg == -1)
2837 m->tx_num_pg = p->tx_num_pgs;
2838 if (m->rx_num_pg == -1)
2839 m->rx_num_pg = p->rx_num_pgs;
2840 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2842 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2843 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2846 p->rx_pg_size = m->rx_pg_sz;
2847 p->tx_pg_size = m->tx_pg_sz;
2848 p->rx_num_pgs = m->rx_num_pg;
2849 p->tx_num_pgs = m->tx_num_pg;
2852 case CHELSIO_SETMTUTAB: {
2853 struct ch_mtus *m = (struct ch_mtus *)data;
2856 if (!is_offload(sc))
2857 return (EOPNOTSUPP);
2858 if (offload_running(sc))
2860 if (m->nmtus != NMTUS)
2862 if (m->mtus[0] < 81) /* accommodate SACK */
2866 * MTUs must be in ascending order
2868 for (i = 1; i < NMTUS; ++i)
2869 if (m->mtus[i] < m->mtus[i - 1])
2872 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2875 case CHELSIO_GETMTUTAB: {
2876 struct ch_mtus *m = (struct ch_mtus *)data;
2878 if (!is_offload(sc))
2879 return (EOPNOTSUPP);
2881 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2885 case CHELSIO_GET_MEM: {
2886 struct ch_mem_range *t = (struct ch_mem_range *)data;
2892 * Use these to avoid modifying len/addr in the the return
2895 uint32_t len = t->len, addr = t->addr;
2897 if (!is_offload(sc))
2898 return (EOPNOTSUPP);
2899 if (!(sc->flags & FULL_INIT_DONE))
2900 return (EIO); /* need the memory controllers */
2901 if ((addr & 0x7) || (len & 0x7))
2903 if (t->mem_id == MEM_CM)
2905 else if (t->mem_id == MEM_PMRX)
2907 else if (t->mem_id == MEM_PMTX)
2914 * bits 0..9: chip version
2915 * bits 10..15: chip revision
2917 t->version = 3 | (sc->params.rev << 10);
2920 * Read 256 bytes at a time as len can be large and we don't
2921 * want to use huge intermediate buffers.
2923 useraddr = (uint8_t *)t->buf;
2925 unsigned int chunk = min(len, sizeof(buf));
2927 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2930 if (copyout(buf, useraddr, chunk))
2938 case CHELSIO_READ_TCAM_WORD: {
2939 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2941 if (!is_offload(sc))
2942 return (EOPNOTSUPP);
2943 if (!(sc->flags & FULL_INIT_DONE))
2944 return (EIO); /* need MC5 */
2945 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2948 case CHELSIO_SET_TRACE_FILTER: {
2949 struct ch_trace *t = (struct ch_trace *)data;
2950 const struct trace_params *tp;
2952 tp = (const struct trace_params *)&t->sip;
2954 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2957 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2961 case CHELSIO_SET_PKTSCHED: {
2962 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2963 if (sc->open_device_map == 0)
2965 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2969 case CHELSIO_IFCONF_GETREGS: {
2970 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2971 int reglen = cxgb_get_regs_len();
2972 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2976 if (regs->len > reglen)
2978 else if (regs->len < reglen)
2982 cxgb_get_regs(sc, regs, buf);
2983 error = copyout(buf, regs->data, reglen);
2985 free(buf, M_DEVBUF);
2989 case CHELSIO_SET_HW_SCHED: {
2990 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2991 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2993 if ((sc->flags & FULL_INIT_DONE) == 0)
2994 return (EAGAIN); /* need TP to be initialized */
2995 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2996 !in_range(t->channel, 0, 1) ||
2997 !in_range(t->kbps, 0, 10000000) ||
2998 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2999 !in_range(t->flow_ipg, 0,
3000 dack_ticks_to_usec(sc, 0x7ff)))
3004 error = t3_config_sched(sc, t->kbps, t->sched);
3008 if (t->class_ipg >= 0)
3009 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
3010 if (t->flow_ipg >= 0) {
3011 t->flow_ipg *= 1000; /* us -> ns */
3012 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
3015 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
3017 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
3018 bit, t->mode ? bit : 0);
3020 if (t->channel >= 0)
3021 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
3022 1 << t->sched, t->channel << t->sched);
3025 case CHELSIO_GET_EEPROM: {
3027 struct ch_eeprom *e = (struct ch_eeprom *)data;
3028 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
3033 e->magic = EEPROM_MAGIC;
3034 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
3035 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
3038 error = copyout(buf + e->offset, e->data, e->len);
3040 free(buf, M_DEVBUF);
3043 case CHELSIO_CLEAR_STATS: {
3044 if (!(sc->flags & FULL_INIT_DONE))
3048 t3_mac_update_stats(&pi->mac);
3049 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
3053 case CHELSIO_GET_UP_LA: {
3054 struct ch_up_la *la = (struct ch_up_la *)data;
3055 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
3059 if (la->bufsize < LA_BUFSIZE)
3063 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
3066 error = copyout(buf, la->data, la->bufsize);
3068 free(buf, M_DEVBUF);
3071 case CHELSIO_GET_UP_IOQS: {
3072 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3073 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3079 if (ioqs->bufsize < IOQS_BUFSIZE)
3083 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3086 v = (uint32_t *)buf;
3088 ioqs->bufsize -= 4 * sizeof(uint32_t);
3089 ioqs->ioq_rx_enable = *v++;
3090 ioqs->ioq_tx_enable = *v++;
3091 ioqs->ioq_rx_status = *v++;
3092 ioqs->ioq_tx_status = *v++;
3094 error = copyout(v, ioqs->data, ioqs->bufsize);
3097 free(buf, M_DEVBUF);
3101 return (EOPNOTSUPP);
3108 static __inline void
3109 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3112 uint32_t *p = (uint32_t *)(buf + start);
3114 for ( ; start <= end; start += sizeof(uint32_t))
3115 *p++ = t3_read_reg(ap, start);
3118 #define T3_REGMAP_SIZE (3 * 1024)
3120 cxgb_get_regs_len(void)
3122 return T3_REGMAP_SIZE;
3126 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3131 * bits 0..9: chip version
3132 * bits 10..15: chip revision
3133 * bit 31: set for PCIe cards
3135 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3138 * We skip the MAC statistics registers because they are clear-on-read.
3139 * Also reading multi-register stats would need to synchronize with the
3140 * periodic mac stats accumulation. Hard to justify the complexity.
3142 memset(buf, 0, cxgb_get_regs_len());
3143 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3144 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3145 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3146 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3147 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3148 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3149 XGM_REG(A_XGM_SERDES_STAT3, 1));
3150 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3151 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3155 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1);