1 /**************************************************************************
3 Copyright (c) 2007-2008, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/pciio.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/bus_dma.h>
45 #include <sys/ioccom.h>
47 #include <sys/linker.h>
48 #include <sys/firmware.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/queue.h>
55 #include <sys/taskqueue.h>
59 #include <net/ethernet.h>
61 #include <net/if_arp.h>
62 #include <net/if_dl.h>
63 #include <net/if_media.h>
64 #include <net/if_types.h>
66 #include <netinet/in_systm.h>
67 #include <netinet/in.h>
68 #include <netinet/if_ether.h>
69 #include <netinet/ip.h>
70 #include <netinet/ip.h>
71 #include <netinet/tcp.h>
72 #include <netinet/udp.h>
74 #include <dev/pci/pcireg.h>
75 #include <dev/pci/pcivar.h>
76 #include <dev/pci/pci_private.h>
79 #include <cxgb_include.h>
81 #include <dev/cxgb/cxgb_include.h>
88 #ifdef IFNET_MULTIQUEUE
89 #include <machine/intr_machdep.h>
92 static int cxgb_setup_msix(adapter_t *, int);
93 static void cxgb_teardown_msix(adapter_t *);
94 static void cxgb_init(void *);
95 static void cxgb_init_locked(struct port_info *);
96 static void cxgb_stop_locked(struct port_info *);
97 static void cxgb_set_rxmode(struct port_info *);
98 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
99 static int cxgb_media_change(struct ifnet *);
100 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
101 static int setup_sge_qsets(adapter_t *);
102 static void cxgb_async_intr(void *);
103 static void cxgb_ext_intr_handler(void *, int);
104 static void cxgb_tick_handler(void *, int);
105 static void cxgb_down_locked(struct adapter *sc);
106 static void cxgb_tick(void *);
107 static void setup_rss(adapter_t *sc);
109 /* Attachment glue for the PCI controller end of the device. Each port of
110 * the device is attached separately, as defined later.
112 static int cxgb_controller_probe(device_t);
113 static int cxgb_controller_attach(device_t);
114 static int cxgb_controller_detach(device_t);
115 static void cxgb_free(struct adapter *);
116 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
118 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
119 static int cxgb_get_regs_len(void);
120 static int offload_open(struct port_info *pi);
121 static void touch_bars(device_t dev);
122 static int offload_close(struct t3cdev *tdev);
123 static void cxgb_link_start(struct port_info *p);
125 static device_method_t cxgb_controller_methods[] = {
126 DEVMETHOD(device_probe, cxgb_controller_probe),
127 DEVMETHOD(device_attach, cxgb_controller_attach),
128 DEVMETHOD(device_detach, cxgb_controller_detach),
131 DEVMETHOD(bus_print_child, bus_generic_print_child),
132 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
137 static driver_t cxgb_controller_driver = {
139 cxgb_controller_methods,
140 sizeof(struct adapter)
143 static devclass_t cxgb_controller_devclass;
144 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
147 * Attachment glue for the ports. Attachment is done directly to the
150 static int cxgb_port_probe(device_t);
151 static int cxgb_port_attach(device_t);
152 static int cxgb_port_detach(device_t);
154 static device_method_t cxgb_port_methods[] = {
155 DEVMETHOD(device_probe, cxgb_port_probe),
156 DEVMETHOD(device_attach, cxgb_port_attach),
157 DEVMETHOD(device_detach, cxgb_port_detach),
161 static driver_t cxgb_port_driver = {
167 static d_ioctl_t cxgb_extension_ioctl;
168 static d_open_t cxgb_extension_open;
169 static d_close_t cxgb_extension_close;
171 static struct cdevsw cxgb_cdevsw = {
172 .d_version = D_VERSION,
174 .d_open = cxgb_extension_open,
175 .d_close = cxgb_extension_close,
176 .d_ioctl = cxgb_extension_ioctl,
180 static devclass_t cxgb_port_devclass;
181 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
183 #define SGE_MSIX_COUNT (SGE_QSETS + 1)
186 * The driver uses the best interrupt scheme available on a platform in the
187 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
188 * of these schemes the driver may consider as follows:
190 * msi = 2: choose from among all three options
191 * msi = 1 : only consider MSI and pin interrupts
192 * msi = 0: force pin interrupts
194 static int msi_allowed = 2;
196 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
197 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
198 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
199 "MSI-X, MSI, INTx selector");
202 * The driver enables offload as a default.
203 * To disable it, use ofld_disable = 1.
205 static int ofld_disable = 0;
206 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
207 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
208 "disable ULP offload");
211 * The driver uses an auto-queue algorithm by default.
212 * To disable it and force a single queue-set per port, use singleq = 1.
214 static int singleq = 0;
215 TUNABLE_INT("hw.cxgb.singleq", &singleq);
216 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
217 "use a single queue-set per port");
221 * The driver uses an auto-queue algorithm by default.
222 * To disable it and force a single queue-set per port, use singleq = 1.
224 static int force_fw_update = 0;
225 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
226 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
227 "update firmware even if up to date");
229 int cxgb_use_16k_clusters = 0;
230 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
231 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
232 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
235 MAX_TXQ_ENTRIES = 16384,
236 MAX_CTRL_TXQ_ENTRIES = 1024,
237 MAX_RSPQ_ENTRIES = 16384,
238 MAX_RX_BUFFERS = 16384,
239 MAX_RX_JUMBO_BUFFERS = 16384,
241 MIN_CTRL_TXQ_ENTRIES = 4,
242 MIN_RSPQ_ENTRIES = 32,
244 MIN_FL_JUMBO_ENTRIES = 32
259 u32 report_filter_id:1;
267 enum { FILTER_NO_VLAN_PRI = 7 };
269 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
271 /* Table for probing the cards. The desc field isn't actually used */
277 } cxgb_identifiers[] = {
278 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
279 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
280 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
281 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
282 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
283 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
284 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
285 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
286 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
287 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
288 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
292 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
296 cxgb_log_tcb(struct adapter *sc, unsigned int tid)
299 uint64_t *tcb = (uint64_t *)buf;
301 struct mc7 *mem = &sc->cm;
303 error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
305 printf("cxgb_tcb_log failed\n");
307 CTR1(KTR_CXGB, "TCB tid=%u", tid);
308 for (i = 0; i < TCB_SIZE / 32; i++) {
309 CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
310 i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
311 (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
313 CTR4(KTR_CXGB, " %08x %08x %08x %08x",
314 (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
315 (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
321 t3rev2char(struct adapter *adapter)
325 switch(adapter->params.rev) {
340 static struct cxgb_ident *
341 cxgb_get_ident(device_t dev)
343 struct cxgb_ident *id;
345 for (id = cxgb_identifiers; id->desc != NULL; id++) {
346 if ((id->vendor == pci_get_vendor(dev)) &&
347 (id->device == pci_get_device(dev))) {
354 static const struct adapter_info *
355 cxgb_get_adapter_info(device_t dev)
357 struct cxgb_ident *id;
358 const struct adapter_info *ai;
360 id = cxgb_get_ident(dev);
364 ai = t3_get_adapter_info(id->index);
370 cxgb_controller_probe(device_t dev)
372 const struct adapter_info *ai;
373 char *ports, buf[80];
376 ai = cxgb_get_adapter_info(dev);
380 nports = ai->nports0 + ai->nports1;
386 snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports);
387 device_set_desc_copy(dev, buf);
388 return (BUS_PROBE_DEFAULT);
391 #define FW_FNAME "cxgb_t3fw"
392 #define TPEEPROM_NAME "t3b_tp_eeprom"
393 #define TPSRAM_NAME "t3b_protocol_sram"
396 upgrade_fw(adapter_t *sc)
398 #ifdef FIRMWARE_LATEST
399 const struct firmware *fw;
405 if ((fw = firmware_get(FW_FNAME)) == NULL) {
406 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
409 device_printf(sc->dev, "updating firmware on card\n");
410 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
412 device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status);
414 firmware_put(fw, FIRMWARE_UNLOAD);
420 cxgb_controller_attach(device_t dev)
423 const struct adapter_info *ai;
432 sc = device_get_softc(dev);
435 ai = cxgb_get_adapter_info(dev);
438 * XXX not really related but a recent addition
441 /* find the PCIe link width and set max read request to 4KB*/
442 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
444 lnk = pci_read_config(dev, reg + 0x12, 2);
445 sc->link_width = (lnk >> 4) & 0x3f;
447 pectl = pci_read_config(dev, reg + 0x8, 2);
448 pectl = (pectl & ~0x7000) | (5 << 12);
449 pci_write_config(dev, reg + 0x8, pectl, 2);
452 if (sc->link_width != 0 && sc->link_width <= 4 &&
453 (ai->nports0 + ai->nports1) <= 2) {
454 device_printf(sc->dev,
455 "PCIe x%d Link, expect reduced performance\n",
460 pci_enable_busmaster(dev);
462 * Allocate the registers and make them available to the driver.
463 * The registers that we care about for NIC mode are in BAR 0
465 sc->regs_rid = PCIR_BAR(0);
466 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
467 &sc->regs_rid, RF_ACTIVE)) == NULL) {
468 device_printf(dev, "Cannot allocate BAR region 0\n");
471 sc->udbs_rid = PCIR_BAR(2);
472 if ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
473 &sc->udbs_rid, RF_ACTIVE)) == NULL) {
474 device_printf(dev, "Cannot allocate BAR region 1\n");
479 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
480 device_get_unit(dev));
481 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
483 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
484 device_get_unit(dev));
485 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
486 device_get_unit(dev));
487 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
488 device_get_unit(dev));
490 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
491 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
492 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
494 sc->bt = rman_get_bustag(sc->regs_res);
495 sc->bh = rman_get_bushandle(sc->regs_res);
496 sc->mmio_len = rman_get_size(sc->regs_res);
498 if (t3_prep_adapter(sc, ai, 1) < 0) {
499 printf("prep adapter failed\n");
503 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
504 * enough messages for the queue sets. If that fails, try falling
505 * back to MSI. If that fails, then try falling back to the legacy
506 * interrupt pin model.
510 sc->msix_regs_rid = 0x20;
511 if ((msi_allowed >= 2) &&
512 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
513 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
515 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
517 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
518 (sc->msi_count != msi_needed)) {
519 device_printf(dev, "msix allocation failed - msi_count = %d"
520 " msi_needed=%d will try msi err=%d\n", sc->msi_count,
523 pci_release_msi(dev);
524 bus_release_resource(dev, SYS_RES_MEMORY,
525 sc->msix_regs_rid, sc->msix_regs_res);
526 sc->msix_regs_res = NULL;
528 sc->flags |= USING_MSIX;
529 sc->cxgb_intr = t3_intr_msix;
533 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
535 if (pci_alloc_msi(dev, &sc->msi_count)) {
536 device_printf(dev, "alloc msi failed - will try INTx\n");
538 pci_release_msi(dev);
540 sc->flags |= USING_MSI;
542 sc->cxgb_intr = t3_intr_msi;
546 if (sc->msi_count == 0) {
547 device_printf(dev, "using line interrupts\n");
549 sc->cxgb_intr = t3b_intr;
552 if ((sc->flags & USING_MSIX) && !singleq)
553 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
555 /* Create a private taskqueue thread for handling driver events */
556 #ifdef TASKQUEUE_CURRENT
557 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
558 taskqueue_thread_enqueue, &sc->tq);
560 sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
561 taskqueue_thread_enqueue, &sc->tq);
563 if (sc->tq == NULL) {
564 device_printf(dev, "failed to allocate controller task queue\n");
568 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
569 device_get_nameunit(dev));
570 TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
571 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
574 /* Create a periodic callout for checking adapter status */
575 callout_init(&sc->cxgb_tick_ch, TRUE);
577 if ((t3_check_fw_version(sc, &must_load) != 0 && must_load) || force_fw_update) {
579 * Warn user that a firmware update will be attempted in init.
581 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
582 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
583 sc->flags &= ~FW_UPTODATE;
585 sc->flags |= FW_UPTODATE;
588 if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) {
590 * Warn user that a firmware update will be attempted in init.
592 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
593 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
594 sc->flags &= ~TPS_UPTODATE;
596 sc->flags |= TPS_UPTODATE;
600 * Create a child device for each MAC. The ethernet attachment
601 * will be done in these children.
603 for (i = 0; i < (sc)->params.nports; i++) {
604 struct port_info *pi;
606 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
607 device_printf(dev, "failed to add child port\n");
613 pi->nqsets = port_qsets;
614 pi->first_qset = i*port_qsets;
616 pi->tx_chan = i >= ai->nports0;
617 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
618 sc->rxpkt_map[pi->txpkt_intf] = i;
619 sc->port[i].tx_chan = i >= ai->nports0;
620 sc->portdev[i] = child;
621 device_set_softc(child, pi);
623 if ((error = bus_generic_attach(dev)) != 0)
627 * XXX need to poll for link status
629 sc->params.stats_update_period = 1;
631 /* initialize sge private state */
632 t3_sge_init_adapter(sc);
637 if (is_offload(sc)) {
638 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
639 cxgb_adapter_ofld(sc);
641 error = t3_get_fw_version(sc, &vers);
645 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
646 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
647 G_FW_VERSION_MICRO(vers));
649 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
650 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
651 t3_add_attach_sysctls(sc);
660 cxgb_controller_detach(device_t dev)
664 sc = device_get_softc(dev);
672 cxgb_free(struct adapter *sc)
677 sc->flags |= CXGB_SHUTDOWN;
679 cxgb_pcpu_shutdown_threads(sc);
685 cxgb_down_locked(sc);
688 if (sc->flags & (USING_MSI | USING_MSIX)) {
689 device_printf(sc->dev, "releasing msi message(s)\n");
690 pci_release_msi(sc->dev);
692 device_printf(sc->dev, "no msi message to release\n");
695 if (sc->msix_regs_res != NULL) {
696 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
700 t3_sge_deinit_sw(sc);
702 * Wait for last callout
707 for (i = 0; i < (sc)->params.nports; ++i) {
708 if (sc->portdev[i] != NULL)
709 device_delete_child(sc->dev, sc->portdev[i]);
712 bus_generic_detach(sc->dev);
713 if (sc->tq != NULL) {
714 taskqueue_free(sc->tq);
718 if (is_offload(sc)) {
719 cxgb_adapter_unofld(sc);
720 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
721 offload_close(&sc->tdev);
723 printf("cxgb_free: DEVMAP_BIT not set\n");
725 printf("not offloading set\n");
727 if (sc->flags & CXGB_OFLD_INIT)
728 cxgb_offload_deactivate(sc);
730 free(sc->filters, M_DEVBUF);
735 if (sc->udbs_res != NULL)
736 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
739 if (sc->regs_res != NULL)
740 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
743 MTX_DESTROY(&sc->mdio_lock);
744 MTX_DESTROY(&sc->sge.reg_lock);
745 MTX_DESTROY(&sc->elmer_lock);
746 ADAPTER_LOCK_DEINIT(sc);
750 * setup_sge_qsets - configure SGE Tx/Rx/response queues
751 * @sc: the controller softc
753 * Determines how many sets of SGE queues to use and initializes them.
754 * We support multiple queue sets per port if we have MSI-X, otherwise
755 * just one queue set per port.
758 setup_sge_qsets(adapter_t *sc)
760 int i, j, err, irq_idx = 0, qset_idx = 0;
761 u_int ntxq = SGE_TXQ_PER_SET;
763 if ((err = t3_sge_alloc(sc)) != 0) {
764 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
768 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
771 for (i = 0; i < (sc)->params.nports; i++) {
772 struct port_info *pi = &sc->port[i];
774 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
775 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
776 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
777 &sc->params.sge.qset[qset_idx], ntxq, pi);
779 t3_free_sge_resources(sc);
780 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n",
791 cxgb_teardown_msix(adapter_t *sc)
795 for (nqsets = i = 0; i < (sc)->params.nports; i++)
796 nqsets += sc->port[i].nqsets;
798 for (i = 0; i < nqsets; i++) {
799 if (sc->msix_intr_tag[i] != NULL) {
800 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
801 sc->msix_intr_tag[i]);
802 sc->msix_intr_tag[i] = NULL;
804 if (sc->msix_irq_res[i] != NULL) {
805 bus_release_resource(sc->dev, SYS_RES_IRQ,
806 sc->msix_irq_rid[i], sc->msix_irq_res[i]);
807 sc->msix_irq_res[i] = NULL;
813 cxgb_setup_msix(adapter_t *sc, int msix_count)
815 int i, j, k, nqsets, rid;
817 /* The first message indicates link changes and error conditions */
819 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
820 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
821 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
825 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
829 cxgb_async_intr, sc, &sc->intr_tag)) {
830 device_printf(sc->dev, "Cannot set up interrupt\n");
833 for (i = k = 0; i < (sc)->params.nports; i++) {
834 nqsets = sc->port[i].nqsets;
835 for (j = 0; j < nqsets; j++, k++) {
836 struct sge_qset *qs = &sc->sge.qs[k];
840 printf("rid=%d ", rid);
841 if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
842 sc->dev, SYS_RES_IRQ, &rid,
843 RF_SHAREABLE | RF_ACTIVE)) == NULL) {
844 device_printf(sc->dev, "Cannot allocate "
845 "interrupt for message %d\n", rid);
848 sc->msix_irq_rid[k] = rid;
849 if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
850 INTR_MPSAFE|INTR_TYPE_NET,
854 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
855 device_printf(sc->dev, "Cannot set up "
856 "interrupt for message %d\n", rid);
859 #ifdef IFNET_MULTIQUEUE
861 int vector = rman_get_start(sc->msix_irq_res[k]);
863 device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus);
864 intr_bind(vector, k % mp_ncpus);
874 cxgb_port_probe(device_t dev)
880 p = device_get_softc(dev);
882 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
883 device_set_desc_copy(dev, buf);
889 cxgb_makedev(struct port_info *pi)
892 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
893 UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
895 if (pi->port_cdev == NULL)
898 pi->port_cdev->si_drv1 = (void *)pi;
905 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU)
906 /* Don't enable TSO6 yet */
907 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU)
909 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
910 /* Don't enable TSO6 yet */
911 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
912 #define IFCAP_TSO4 0x0
913 #define IFCAP_TSO6 0x0
919 cxgb_port_attach(device_t dev)
923 int err, media_flags;
927 p = device_get_softc(dev);
929 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
930 device_get_unit(device_get_parent(dev)), p->port_id);
931 PORT_LOCK_INIT(p, p->lockbuf);
933 /* Allocate an ifnet object and set it up */
934 ifp = p->ifp = if_alloc(IFT_ETHER);
936 device_printf(dev, "Cannot allocate ifnet\n");
941 * Note that there is currently no watchdog timer.
943 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
944 ifp->if_init = cxgb_init;
946 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
947 ifp->if_ioctl = cxgb_ioctl;
948 ifp->if_start = cxgb_start;
951 #ifdef IFNET_MULTIQUEUE
952 ifp->if_flags |= IFF_MULTIQ;
953 ifp->if_mq_start = cxgb_pcpu_start;
956 ifp->if_timer = 0; /* Disable ifnet watchdog */
957 ifp->if_watchdog = NULL;
959 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
960 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
961 IFQ_SET_READY(&ifp->if_snd);
963 ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
964 ifp->if_capabilities |= CXGB_CAP;
965 ifp->if_capenable |= CXGB_CAP_ENABLE;
966 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
968 * disable TSO on 4-port - it isn't supported by the firmware yet
970 if (p->adapter->params.nports > 2) {
971 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6);
972 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6);
973 ifp->if_hwassist &= ~CSUM_TSO;
976 ether_ifattach(ifp, p->hw_addr);
978 * Only default to jumbo frames on 10GigE
980 if (p->adapter->params.nports <= 2)
982 if ((err = cxgb_makedev(p)) != 0) {
983 printf("makedev failed %d\n", err);
986 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
989 if (!strcmp(p->phy.desc, "10GBASE-CX4")) {
990 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
991 } else if (!strcmp(p->phy.desc, "10GBASE-SR")) {
992 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
993 } else if (!strcmp(p->phy.desc, "10GBASE-R")) {
994 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
995 } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) {
996 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
997 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
999 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
1001 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1003 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1007 printf("unsupported media type %s\n", p->phy.desc);
1011 ifmedia_add(&p->media, media_flags, 0, NULL);
1012 ifmedia_set(&p->media, media_flags);
1014 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1015 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
1019 snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port_id);
1020 #ifdef TASKQUEUE_CURRENT
1021 /* Create a port for handling TX without starvation */
1022 p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT,
1023 taskqueue_thread_enqueue, &p->tq);
1025 /* Create a port for handling TX without starvation */
1026 p->tq = taskqueue_create_fast(p->taskqbuf, M_NOWAIT,
1027 taskqueue_thread_enqueue, &p->tq);
1029 /* Get the latest mac address, User can use a LAA */
1030 bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN);
1031 t3_sge_init_port(p);
1032 #if defined(LINK_ATTACH)
1034 t3_link_changed(sc, p->port_id);
1040 cxgb_port_detach(device_t dev)
1042 struct port_info *p;
1044 p = device_get_softc(dev);
1047 if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
1048 cxgb_stop_locked(p);
1051 if (p->tq != NULL) {
1052 taskqueue_drain(p->tq, &p->start_task);
1053 taskqueue_free(p->tq);
1057 ether_ifdetach(p->ifp);
1058 printf("waiting for callout to stop ...");
1062 * the lock may be acquired in ifdetach
1064 PORT_LOCK_DEINIT(p);
1067 if (p->port_cdev != NULL)
1068 destroy_dev(p->port_cdev);
1074 t3_fatal_err(struct adapter *sc)
1078 if (sc->flags & FULL_INIT_DONE) {
1080 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1081 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1082 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1083 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1084 t3_intr_disable(sc);
1086 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1087 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1088 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1089 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1093 t3_os_find_pci_capability(adapter_t *sc, int cap)
1096 struct pci_devinfo *dinfo;
1102 dinfo = device_get_ivars(dev);
1105 status = pci_read_config(dev, PCIR_STATUS, 2);
1106 if (!(status & PCIM_STATUS_CAPPRESENT))
1109 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1115 ptr = PCIR_CAP_PTR_2;
1121 ptr = pci_read_config(dev, ptr, 1);
1124 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1126 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1133 t3_os_pci_save_state(struct adapter *sc)
1136 struct pci_devinfo *dinfo;
1139 dinfo = device_get_ivars(dev);
1141 pci_cfg_save(dev, dinfo, 0);
1146 t3_os_pci_restore_state(struct adapter *sc)
1149 struct pci_devinfo *dinfo;
1152 dinfo = device_get_ivars(dev);
1154 pci_cfg_restore(dev, dinfo);
1159 * t3_os_link_changed - handle link status changes
1160 * @adapter: the adapter associated with the link change
1161 * @port_id: the port index whose limk status has changed
1162 * @link_status: the new status of the link
1163 * @speed: the new speed setting
1164 * @duplex: the new duplex setting
1165 * @fc: the new flow-control setting
1167 * This is the OS-dependent handler for link status changes. The OS
1168 * neutral handler takes care of most of the processing for these events,
1169 * then calls this handler for any OS-specific processing.
1172 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1175 struct port_info *pi = &adapter->port[port_id];
1176 struct cmac *mac = &adapter->port[port_id].mac;
1180 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1181 /* Clear errors created by MAC enable */
1182 t3_set_reg_field(adapter,
1183 A_XGM_STAT_CTRL + pi->mac.offset,
1185 if_link_state_change(pi->ifp, LINK_STATE_UP);
1188 pi->phy.ops->power_down(&pi->phy, 1);
1189 t3_mac_disable(mac, MAC_DIRECTION_RX);
1190 t3_link_start(&pi->phy, mac, &pi->link_config);
1191 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1192 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
1197 * Interrupt-context handler for external (PHY) interrupts.
1200 t3_os_ext_intr_handler(adapter_t *sc)
1203 printf("t3_os_ext_intr_handler\n");
1205 * Schedule a task to handle external interrupts as they may be slow
1206 * and we use a mutex to protect MDIO registers. We disable PHY
1207 * interrupts in the meantime and let the task reenable them when
1211 if (sc->slow_intr_mask) {
1212 sc->slow_intr_mask &= ~F_T3DBG;
1213 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1214 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1220 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1224 * The ifnet might not be allocated before this gets called,
1225 * as this is called early on in attach by t3_prep_adapter
1226 * save the address off in the port structure
1229 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1230 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1234 * link_start - enable a port
1235 * @p: the port to enable
1237 * Performs the MAC and PHY actions needed to enable a port.
1240 cxgb_link_start(struct port_info *p)
1243 struct t3_rx_mode rm;
1244 struct cmac *mac = &p->mac;
1248 t3_init_rx_mode(&rm, p);
1249 if (!mac->multiport)
1251 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1252 t3_mac_set_address(mac, 0, p->hw_addr);
1253 t3_mac_set_rx_mode(mac, &rm);
1254 t3_link_start(&p->phy, mac, &p->link_config);
1255 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1260 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1265 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1274 init_tp_parity(struct adapter *adap)
1278 struct cpl_set_tcb_field *greq;
1279 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1281 t3_tp_set_offload_mode(adap, 1);
1283 for (i = 0; i < 16; i++) {
1284 struct cpl_smt_write_req *req;
1286 m = m_gethdr(M_WAITOK, MT_DATA);
1287 req = mtod(m, struct cpl_smt_write_req *);
1288 m->m_len = m->m_pkthdr.len = sizeof(*req);
1289 memset(req, 0, sizeof(*req));
1290 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1291 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1293 t3_mgmt_tx(adap, m);
1296 for (i = 0; i < 2048; i++) {
1297 struct cpl_l2t_write_req *req;
1299 m = m_gethdr(M_WAITOK, MT_DATA);
1300 req = mtod(m, struct cpl_l2t_write_req *);
1301 m->m_len = m->m_pkthdr.len = sizeof(*req);
1302 memset(req, 0, sizeof(*req));
1303 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1304 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1305 req->params = htonl(V_L2T_W_IDX(i));
1306 t3_mgmt_tx(adap, m);
1309 for (i = 0; i < 2048; i++) {
1310 struct cpl_rte_write_req *req;
1312 m = m_gethdr(M_WAITOK, MT_DATA);
1313 req = mtod(m, struct cpl_rte_write_req *);
1314 m->m_len = m->m_pkthdr.len = sizeof(*req);
1315 memset(req, 0, sizeof(*req));
1316 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1317 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1318 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1319 t3_mgmt_tx(adap, m);
1322 m = m_gethdr(M_WAITOK, MT_DATA);
1323 greq = mtod(m, struct cpl_set_tcb_field *);
1324 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1325 memset(greq, 0, sizeof(*greq));
1326 greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1327 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1328 greq->mask = htobe64(1);
1329 t3_mgmt_tx(adap, m);
1331 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1332 t3_tp_set_offload_mode(adap, 0);
1337 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1338 * @adap: the adapter
1340 * Sets up RSS to distribute packets to multiple receive queues. We
1341 * configure the RSS CPU lookup table to distribute to the number of HW
1342 * receive queues, and the response queue lookup table to narrow that
1343 * down to the response queues actually configured for each port.
1344 * We always configure the RSS mapping for two ports since the mapping
1345 * table has plenty of entries.
1348 setup_rss(adapter_t *adap)
1352 uint8_t cpus[SGE_QSETS + 1];
1353 uint16_t rspq_map[RSS_TABLE_SIZE];
1355 for (i = 0; i < SGE_QSETS; ++i)
1357 cpus[SGE_QSETS] = 0xff;
1360 for_each_port(adap, i) {
1361 const struct port_info *pi = adap2pinfo(adap, i);
1363 nq[pi->tx_chan] += pi->nqsets;
1365 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1366 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1367 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1369 /* Calculate the reverse RSS map table */
1370 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1371 if (adap->rrss_map[rspq_map[i]] == 0xff)
1372 adap->rrss_map[rspq_map[i]] = i;
1374 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1375 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1376 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1382 * Sends an mbuf to an offload queue driver
1383 * after dealing with any active network taps.
1386 offload_tx(struct t3cdev *tdev, struct mbuf *m)
1390 ret = t3_offload_tx(tdev, m);
1395 write_smt_entry(struct adapter *adapter, int idx)
1397 struct port_info *pi = &adapter->port[idx];
1398 struct cpl_smt_write_req *req;
1401 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1404 req = mtod(m, struct cpl_smt_write_req *);
1405 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
1407 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1408 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1409 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1411 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1412 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1414 m_set_priority(m, 1);
1416 offload_tx(&adapter->tdev, m);
1422 init_smt(struct adapter *adapter)
1426 for_each_port(adapter, i)
1427 write_smt_entry(adapter, i);
1432 init_port_mtus(adapter_t *adapter)
1434 unsigned int mtus = adapter->port[0].ifp->if_mtu;
1436 if (adapter->port[1].ifp)
1437 mtus |= adapter->port[1].ifp->if_mtu << 16;
1438 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1442 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1446 struct mngt_pktsched_wr *req;
1448 m = m_gethdr(M_DONTWAIT, MT_DATA);
1450 req = mtod(m, struct mngt_pktsched_wr *);
1451 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1452 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1457 req->binding = port;
1458 m->m_len = m->m_pkthdr.len = sizeof(*req);
1459 t3_mgmt_tx(adap, m);
1464 bind_qsets(adapter_t *sc)
1468 cxgb_pcpu_startup_threads(sc);
1469 for (i = 0; i < (sc)->params.nports; ++i) {
1470 const struct port_info *pi = adap2pinfo(sc, i);
1472 for (j = 0; j < pi->nqsets; ++j) {
1473 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1481 update_tpeeprom(struct adapter *adap)
1483 #ifdef FIRMWARE_LATEST
1484 const struct firmware *tpeeprom;
1486 struct firmware *tpeeprom;
1490 unsigned int major, minor;
1494 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1496 major = G_TP_VERSION_MAJOR(version);
1497 minor = G_TP_VERSION_MINOR(version);
1498 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1501 rev = t3rev2char(adap);
1503 tpeeprom = firmware_get(TPEEPROM_NAME);
1504 if (tpeeprom == NULL) {
1505 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n",
1510 len = tpeeprom->datasize - 4;
1512 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1514 goto release_tpeeprom;
1516 if (len != TP_SRAM_LEN) {
1517 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", TPEEPROM_NAME, len, TP_SRAM_LEN);
1521 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1525 device_printf(adap->dev,
1526 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1527 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1529 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n");
1532 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1538 update_tpsram(struct adapter *adap)
1540 #ifdef FIRMWARE_LATEST
1541 const struct firmware *tpsram;
1543 struct firmware *tpsram;
1548 rev = t3rev2char(adap);
1552 update_tpeeprom(adap);
1554 tpsram = firmware_get(TPSRAM_NAME);
1555 if (tpsram == NULL){
1556 device_printf(adap->dev, "could not load TP SRAM\n");
1559 device_printf(adap->dev, "updating TP SRAM\n");
1561 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1563 goto release_tpsram;
1565 ret = t3_set_proto_sram(adap, tpsram->data);
1567 device_printf(adap->dev, "loading protocol SRAM failed\n");
1570 firmware_put(tpsram, FIRMWARE_UNLOAD);
1576 * cxgb_up - enable the adapter
1577 * @adap: adapter being enabled
1579 * Called when the first port is enabled, this function performs the
1580 * actions necessary to make an adapter operational, such as completing
1581 * the initialization of HW modules, and enabling interrupts.
1585 cxgb_up(struct adapter *sc)
1589 if ((sc->flags & FULL_INIT_DONE) == 0) {
1591 if ((sc->flags & FW_UPTODATE) == 0)
1592 if ((err = upgrade_fw(sc)))
1594 if ((sc->flags & TPS_UPTODATE) == 0)
1595 if ((err = update_tpsram(sc)))
1597 err = t3_init_hw(sc, 0);
1601 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1602 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1604 err = setup_sge_qsets(sc);
1609 t3_add_configured_sysctls(sc);
1610 sc->flags |= FULL_INIT_DONE;
1615 /* If it's MSI or INTx, allocate a single interrupt for everything */
1616 if ((sc->flags & USING_MSIX) == 0) {
1617 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1618 &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1619 device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n",
1624 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1626 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1630 sc->cxgb_intr, sc, &sc->intr_tag)) {
1631 device_printf(sc->dev, "Cannot set up interrupt\n");
1636 cxgb_setup_msix(sc, sc->msi_count);
1642 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1643 is_offload(sc) && init_tp_parity(sc) == 0)
1644 sc->flags |= TP_PARITY_INIT;
1646 if (sc->flags & TP_PARITY_INIT) {
1647 t3_write_reg(sc, A_TP_INT_CAUSE,
1648 F_CMCACHEPERR | F_ARPLUTPERR);
1649 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1653 if (!(sc->flags & QUEUES_BOUND)) {
1655 sc->flags |= QUEUES_BOUND;
1660 CH_ERR(sc, "request_irq failed, err %d\n", err);
1666 * Release resources when all the ports and offloading have been stopped.
1669 cxgb_down_locked(struct adapter *sc)
1673 t3_intr_disable(sc);
1675 if (sc->intr_tag != NULL) {
1676 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1677 sc->intr_tag = NULL;
1679 if (sc->irq_res != NULL) {
1680 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1681 sc->irq_rid, sc->irq_res);
1682 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1687 if (sc->flags & USING_MSIX)
1688 cxgb_teardown_msix(sc);
1690 callout_stop(&sc->cxgb_tick_ch);
1691 callout_stop(&sc->sge_timer_ch);
1692 callout_drain(&sc->cxgb_tick_ch);
1693 callout_drain(&sc->sge_timer_ch);
1695 if (sc->tq != NULL) {
1696 printf("draining slow intr\n");
1698 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1699 printf("draining ext intr\n");
1700 taskqueue_drain(sc->tq, &sc->ext_intr_task);
1701 printf("draining tick task\n");
1702 taskqueue_drain(sc->tq, &sc->tick_task);
1708 offload_open(struct port_info *pi)
1710 struct adapter *adapter = pi->adapter;
1711 struct t3cdev *tdev = &adapter->tdev;
1715 int adap_up = adapter->open_device_map & PORT_MASK;
1718 CTR1(KTR_CXGB, "device_map=0x%x", adapter->open_device_map);
1719 if (atomic_cmpset_int(&adapter->open_device_map,
1720 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)),
1721 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0)
1725 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1726 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", adapter->open_device_map);
1727 ADAPTER_LOCK(pi->adapter);
1729 err = cxgb_up(adapter);
1730 ADAPTER_UNLOCK(pi->adapter);
1734 t3_tp_set_offload_mode(adapter, 1);
1735 tdev->lldev = pi->ifp;
1737 init_port_mtus(adapter);
1738 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1739 adapter->params.b_wnd,
1740 adapter->params.rev == 0 ?
1741 adapter->port[0].ifp->if_mtu : 0xffff);
1744 /* Call back all registered clients */
1745 cxgb_add_clients(tdev);
1748 /* restore them in case the offload module has changed them */
1750 t3_tp_set_offload_mode(adapter, 0);
1751 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1752 cxgb_set_dummy_ops(tdev);
1758 offload_close(struct t3cdev *tdev)
1760 struct adapter *adapter = tdev2adap(tdev);
1762 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1765 /* Call back all registered clients */
1766 cxgb_remove_clients(tdev);
1769 cxgb_set_dummy_ops(tdev);
1770 t3_tp_set_offload_mode(adapter, 0);
1771 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1773 ADAPTER_LOCK(adapter);
1774 if (!adapter->open_device_map)
1775 cxgb_down_locked(adapter);
1777 ADAPTER_UNLOCK(adapter);
1783 cxgb_init(void *arg)
1785 struct port_info *p = arg;
1788 cxgb_init_locked(p);
1793 cxgb_init_locked(struct port_info *p)
1796 adapter_t *sc = p->adapter;
1799 PORT_LOCK_ASSERT_OWNED(p);
1802 ADAPTER_LOCK(p->adapter);
1803 if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) {
1804 ADAPTER_UNLOCK(p->adapter);
1805 cxgb_stop_locked(p);
1808 if (p->adapter->open_device_map == 0) {
1811 setbit(&p->adapter->open_device_map, p->port_id);
1812 ADAPTER_UNLOCK(p->adapter);
1814 if (is_offload(sc) && !ofld_disable) {
1815 err = offload_open(p);
1818 "Could not initialize offload capabilities\n");
1820 #if !defined(LINK_ATTACH)
1822 t3_link_changed(sc, p->port_id);
1824 ifp->if_baudrate = p->link_config.speed * 1000000;
1826 device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
1827 t3_port_intr_enable(sc, p->port_id);
1829 t3_sge_reset_adapter(sc);
1831 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1832 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1836 cxgb_set_rxmode(struct port_info *p)
1838 struct t3_rx_mode rm;
1839 struct cmac *mac = &p->mac;
1841 t3_init_rx_mode(&rm, p);
1842 mtx_lock(&p->adapter->mdio_lock);
1843 t3_mac_set_rx_mode(mac, &rm);
1844 mtx_unlock(&p->adapter->mdio_lock);
1848 cxgb_stop_locked(struct port_info *pi)
1852 PORT_LOCK_ASSERT_OWNED(pi);
1853 ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter);
1856 t3_port_intr_disable(pi->adapter, pi->port_id);
1857 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1859 /* disable pause frames */
1860 t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset,
1863 /* Reset RX FIFO HWM */
1864 t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG + pi->mac.offset,
1865 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1868 ADAPTER_LOCK(pi->adapter);
1869 clrbit(&pi->adapter->open_device_map, pi->port_id);
1871 if (pi->adapter->open_device_map == 0) {
1872 cxgb_down_locked(pi->adapter);
1874 ADAPTER_UNLOCK(pi->adapter);
1876 #if !defined(LINK_ATTACH)
1879 /* Wait for TXFIFO empty */
1880 t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
1881 F_TXFIFO_EMPTY, 1, 20, 5);
1884 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1886 pi->phy.ops->power_down(&pi->phy, 1);
1892 cxgb_set_mtu(struct port_info *p, int mtu)
1894 struct ifnet *ifp = p->ifp;
1897 if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO))
1899 else if (ifp->if_mtu != mtu) {
1902 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1903 cxgb_stop_locked(p);
1904 cxgb_init_locked(p);
1912 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1914 struct port_info *p = ifp->if_softc;
1915 struct ifaddr *ifa = (struct ifaddr *)data;
1916 struct ifreq *ifr = (struct ifreq *)data;
1917 int flags, error = 0;
1921 * XXX need to check that we aren't in the middle of an unload
1925 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1928 if (ifa->ifa_addr->sa_family == AF_INET) {
1929 ifp->if_flags |= IFF_UP;
1930 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1932 cxgb_init_locked(p);
1935 arp_ifinit(ifp, ifa);
1937 error = ether_ioctl(ifp, command, data);
1941 if (ifp->if_flags & IFF_UP) {
1942 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1943 flags = p->if_flags;
1944 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1945 ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
1948 cxgb_init_locked(p);
1949 p->if_flags = ifp->if_flags;
1950 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1951 cxgb_stop_locked(p);
1957 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1963 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
1967 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1968 if (mask & IFCAP_TXCSUM) {
1969 if (IFCAP_TXCSUM & ifp->if_capenable) {
1970 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
1971 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
1974 ifp->if_capenable |= IFCAP_TXCSUM;
1975 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1977 } else if (mask & IFCAP_RXCSUM) {
1978 if (IFCAP_RXCSUM & ifp->if_capenable) {
1979 ifp->if_capenable &= ~IFCAP_RXCSUM;
1981 ifp->if_capenable |= IFCAP_RXCSUM;
1984 if (mask & IFCAP_TSO4) {
1985 if (IFCAP_TSO4 & ifp->if_capenable) {
1986 ifp->if_capenable &= ~IFCAP_TSO4;
1987 ifp->if_hwassist &= ~CSUM_TSO;
1988 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
1989 ifp->if_capenable |= IFCAP_TSO4;
1990 ifp->if_hwassist |= CSUM_TSO;
1993 printf("cxgb requires tx checksum offload"
1994 " be enabled to use TSO\n");
2001 error = ether_ioctl(ifp, command, data);
2008 cxgb_media_change(struct ifnet *ifp)
2010 if_printf(ifp, "media change not supported\n");
2015 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2017 struct port_info *p = ifp->if_softc;
2019 ifmr->ifm_status = IFM_AVALID;
2020 ifmr->ifm_active = IFM_ETHER;
2022 if (!p->link_config.link_ok)
2025 ifmr->ifm_status |= IFM_ACTIVE;
2027 switch (p->link_config.speed) {
2029 ifmr->ifm_active |= IFM_10_T;
2032 ifmr->ifm_active |= IFM_100_TX;
2035 ifmr->ifm_active |= IFM_1000_T;
2039 if (p->link_config.duplex)
2040 ifmr->ifm_active |= IFM_FDX;
2042 ifmr->ifm_active |= IFM_HDX;
2046 cxgb_async_intr(void *data)
2048 adapter_t *sc = data;
2051 device_printf(sc->dev, "cxgb_async_intr\n");
2053 * May need to sleep - defer to taskqueue
2055 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2059 cxgb_ext_intr_handler(void *arg, int count)
2061 adapter_t *sc = (adapter_t *)arg;
2064 printf("cxgb_ext_intr_handler\n");
2066 t3_phy_intr_handler(sc);
2068 /* Now reenable external interrupts */
2070 if (sc->slow_intr_mask) {
2071 sc->slow_intr_mask |= F_T3DBG;
2072 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
2073 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
2079 check_link_status(adapter_t *sc)
2083 for (i = 0; i < (sc)->params.nports; ++i) {
2084 struct port_info *p = &sc->port[i];
2086 if (!(p->phy.caps & SUPPORTED_IRQ))
2087 t3_link_changed(sc, i);
2088 p->ifp->if_baudrate = p->link_config.speed * 1000000;
2093 check_t3b2_mac(struct adapter *adapter)
2097 if(adapter->flags & CXGB_SHUTDOWN)
2100 for_each_port(adapter, i) {
2101 struct port_info *p = &adapter->port[i];
2102 struct ifnet *ifp = p->ifp;
2105 if(adapter->flags & CXGB_SHUTDOWN)
2108 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2113 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
2114 status = t3b2_mac_watchdog_task(&p->mac);
2116 p->mac.stats.num_toggled++;
2117 else if (status == 2) {
2118 struct cmac *mac = &p->mac;
2120 t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN
2121 + ETHER_VLAN_ENCAP_LEN);
2122 t3_mac_set_address(mac, 0, p->hw_addr);
2124 t3_link_start(&p->phy, mac, &p->link_config);
2125 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2126 t3_port_intr_enable(adapter, p->port_id);
2127 p->mac.stats.num_resets++;
2134 cxgb_tick(void *arg)
2136 adapter_t *sc = (adapter_t *)arg;
2138 if(sc->flags & CXGB_SHUTDOWN)
2141 taskqueue_enqueue(sc->tq, &sc->tick_task);
2142 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2146 cxgb_tick_handler(void *arg, int count)
2148 adapter_t *sc = (adapter_t *)arg;
2149 const struct adapter_params *p = &sc->params;
2151 if(sc->flags & CXGB_SHUTDOWN)
2155 if (p->linkpoll_period)
2156 check_link_status(sc);
2159 * adapter lock can currently only be acquired after the
2164 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2169 touch_bars(device_t dev)
2174 #if !defined(__LP64__) && 0
2177 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2178 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2179 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2180 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2181 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2182 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2187 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2191 u32 aligned_offset, aligned_len, *p;
2192 struct adapter *adapter = pi->adapter;
2195 aligned_offset = offset & ~3;
2196 aligned_len = (len + (offset & 3) + 3) & ~3;
2198 if (aligned_offset != offset || aligned_len != len) {
2199 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2202 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2203 if (!err && aligned_len > 4)
2204 err = t3_seeprom_read(adapter,
2205 aligned_offset + aligned_len - 4,
2206 (u32 *)&buf[aligned_len - 4]);
2209 memcpy(buf + (offset & 3), data, len);
2211 buf = (uint8_t *)(uintptr_t)data;
2213 err = t3_seeprom_wp(adapter, 0);
2217 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2218 err = t3_seeprom_write(adapter, aligned_offset, *p);
2219 aligned_offset += 4;
2223 err = t3_seeprom_wp(adapter, 1);
2226 free(buf, M_DEVBUF);
2232 in_range(int val, int lo, int hi)
2234 return val < 0 || (val <= hi && val >= lo);
2238 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
2244 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
2250 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2251 int fflag, struct thread *td)
2254 struct port_info *pi = dev->si_drv1;
2255 adapter_t *sc = pi->adapter;
2257 #ifdef PRIV_SUPPORTED
2258 if (priv_check(td, PRIV_DRIVER)) {
2260 printf("user does not have access to privileged ioctls\n");
2266 printf("user does not have access to privileged ioctls\n");
2274 struct cphy *phy = &pi->phy;
2275 struct mii_data *mid = (struct mii_data *)data;
2277 if (!phy->mdio_read)
2278 return (EOPNOTSUPP);
2280 mmd = mid->phy_id >> 8;
2283 else if (mmd > MDIO_DEV_XGXS)
2286 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2287 mid->reg_num, &val);
2289 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2290 mid->reg_num & 0x1f, &val);
2296 struct cphy *phy = &pi->phy;
2297 struct mii_data *mid = (struct mii_data *)data;
2299 if (!phy->mdio_write)
2300 return (EOPNOTSUPP);
2302 mmd = mid->phy_id >> 8;
2305 else if (mmd > MDIO_DEV_XGXS)
2308 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2309 mmd, mid->reg_num, mid->val_in);
2311 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2312 mid->reg_num & 0x1f,
2316 case CHELSIO_SETREG: {
2317 struct ch_reg *edata = (struct ch_reg *)data;
2318 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2320 t3_write_reg(sc, edata->addr, edata->val);
2323 case CHELSIO_GETREG: {
2324 struct ch_reg *edata = (struct ch_reg *)data;
2325 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2327 edata->val = t3_read_reg(sc, edata->addr);
2330 case CHELSIO_GET_SGE_CONTEXT: {
2331 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2332 mtx_lock_spin(&sc->sge.reg_lock);
2333 switch (ecntxt->cntxt_type) {
2334 case CNTXT_TYPE_EGRESS:
2335 error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2339 error = t3_sge_read_fl(sc, ecntxt->cntxt_id,
2342 case CNTXT_TYPE_RSP:
2343 error = t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2347 error = t3_sge_read_cq(sc, ecntxt->cntxt_id,
2354 mtx_unlock_spin(&sc->sge.reg_lock);
2357 case CHELSIO_GET_SGE_DESC: {
2358 struct ch_desc *edesc = (struct ch_desc *)data;
2360 if (edesc->queue_num >= SGE_QSETS * 6)
2362 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2363 edesc->queue_num % 6, edesc->idx, edesc->data);
2369 case CHELSIO_SET_QSET_PARAMS: {
2370 struct qset_params *q;
2371 struct ch_qset_params *t = (struct ch_qset_params *)data;
2374 if (t->qset_idx >= SGE_QSETS)
2376 if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
2377 !in_range(t->cong_thres, 0, 255) ||
2378 !in_range(t->txq_size[0], MIN_TXQ_ENTRIES,
2380 !in_range(t->txq_size[1], MIN_TXQ_ENTRIES,
2382 !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2383 MAX_CTRL_TXQ_ENTRIES) ||
2384 !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) ||
2385 !in_range(t->fl_size[1], MIN_FL_ENTRIES,
2386 MAX_RX_JUMBO_BUFFERS) ||
2387 !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
2390 if ((sc->flags & FULL_INIT_DONE) && t->lro > 0)
2391 for_each_port(sc, i) {
2392 pi = adap2pinfo(sc, i);
2393 if (t->qset_idx >= pi->first_qset &&
2394 t->qset_idx < pi->first_qset + pi->nqsets
2396 && !pi->rx_csum_offload
2401 if ((sc->flags & FULL_INIT_DONE) &&
2402 (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
2403 t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||
2404 t->txq_size[1] >= 0 || t->txq_size[2] >= 0 ||
2405 t->polling >= 0 || t->cong_thres >= 0))
2408 q = &sc->params.sge.qset[t->qset_idx];
2410 if (t->rspq_size >= 0)
2411 q->rspq_size = t->rspq_size;
2412 if (t->fl_size[0] >= 0)
2413 q->fl_size = t->fl_size[0];
2414 if (t->fl_size[1] >= 0)
2415 q->jumbo_size = t->fl_size[1];
2416 if (t->txq_size[0] >= 0)
2417 q->txq_size[0] = t->txq_size[0];
2418 if (t->txq_size[1] >= 0)
2419 q->txq_size[1] = t->txq_size[1];
2420 if (t->txq_size[2] >= 0)
2421 q->txq_size[2] = t->txq_size[2];
2422 if (t->cong_thres >= 0)
2423 q->cong_thres = t->cong_thres;
2424 if (t->intr_lat >= 0) {
2425 struct sge_qset *qs = &sc->sge.qs[t->qset_idx];
2427 q->coalesce_nsecs = t->intr_lat*1000;
2428 t3_update_qset_coalesce(qs, q);
2432 case CHELSIO_GET_QSET_PARAMS: {
2433 struct qset_params *q;
2434 struct ch_qset_params *t = (struct ch_qset_params *)data;
2436 if (t->qset_idx >= SGE_QSETS)
2439 q = &(sc)->params.sge.qset[t->qset_idx];
2440 t->rspq_size = q->rspq_size;
2441 t->txq_size[0] = q->txq_size[0];
2442 t->txq_size[1] = q->txq_size[1];
2443 t->txq_size[2] = q->txq_size[2];
2444 t->fl_size[0] = q->fl_size;
2445 t->fl_size[1] = q->jumbo_size;
2446 t->polling = q->polling;
2447 t->intr_lat = q->coalesce_nsecs / 1000;
2448 t->cong_thres = q->cong_thres;
2451 case CHELSIO_SET_QSET_NUM: {
2452 struct ch_reg *edata = (struct ch_reg *)data;
2453 unsigned int port_idx = pi->port_id;
2455 if (sc->flags & FULL_INIT_DONE)
2457 if (edata->val < 1 ||
2458 (edata->val > 1 && !(sc->flags & USING_MSIX)))
2460 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
2462 sc->port[port_idx].nqsets = edata->val;
2463 sc->port[0].first_qset = 0;
2465 * XXX hardcode ourselves to 2 ports just like LEEENUX
2467 sc->port[1].first_qset = sc->port[0].nqsets;
2470 case CHELSIO_GET_QSET_NUM: {
2471 struct ch_reg *edata = (struct ch_reg *)data;
2472 edata->val = pi->nqsets;
2476 case CHELSIO_LOAD_FW:
2477 case CHELSIO_GET_PM:
2478 case CHELSIO_SET_PM:
2479 return (EOPNOTSUPP);
2482 case CHELSIO_SETMTUTAB: {
2483 struct ch_mtus *m = (struct ch_mtus *)data;
2486 if (!is_offload(sc))
2487 return (EOPNOTSUPP);
2488 if (offload_running(sc))
2490 if (m->nmtus != NMTUS)
2492 if (m->mtus[0] < 81) /* accommodate SACK */
2496 * MTUs must be in ascending order
2498 for (i = 1; i < NMTUS; ++i)
2499 if (m->mtus[i] < m->mtus[i - 1])
2502 memcpy(sc->params.mtus, m->mtus,
2503 sizeof(sc->params.mtus));
2506 case CHELSIO_GETMTUTAB: {
2507 struct ch_mtus *m = (struct ch_mtus *)data;
2509 if (!is_offload(sc))
2510 return (EOPNOTSUPP);
2512 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2517 if (!is_offload(sc))
2518 return (EOPNOTSUPP);
2519 return offload_open(pi);
2521 case CHELSIO_GET_MEM: {
2522 struct ch_mem_range *t = (struct ch_mem_range *)data;
2527 if (!is_offload(sc))
2528 return (EOPNOTSUPP);
2529 if (!(sc->flags & FULL_INIT_DONE))
2530 return (EIO); /* need the memory controllers */
2531 if ((t->addr & 0x7) || (t->len & 0x7))
2533 if (t->mem_id == MEM_CM)
2535 else if (t->mem_id == MEM_PMRX)
2537 else if (t->mem_id == MEM_PMTX)
2544 * bits 0..9: chip version
2545 * bits 10..15: chip revision
2547 t->version = 3 | (sc->params.rev << 10);
2550 * Read 256 bytes at a time as len can be large and we don't
2551 * want to use huge intermediate buffers.
2553 useraddr = (uint8_t *)t->buf;
2555 unsigned int chunk = min(t->len, sizeof(buf));
2557 error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf);
2560 if (copyout(buf, useraddr, chunk))
2568 case CHELSIO_READ_TCAM_WORD: {
2569 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2571 if (!is_offload(sc))
2572 return (EOPNOTSUPP);
2573 if (!(sc->flags & FULL_INIT_DONE))
2574 return (EIO); /* need MC5 */
2575 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2578 case CHELSIO_SET_TRACE_FILTER: {
2579 struct ch_trace *t = (struct ch_trace *)data;
2580 const struct trace_params *tp;
2582 tp = (const struct trace_params *)&t->sip;
2584 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2587 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2591 case CHELSIO_SET_PKTSCHED: {
2592 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2593 if (sc->open_device_map == 0)
2595 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2599 case CHELSIO_IFCONF_GETREGS: {
2600 struct ifconf_regs *regs = (struct ifconf_regs *)data;
2601 int reglen = cxgb_get_regs_len();
2602 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT);
2605 } if (regs->len > reglen)
2607 else if (regs->len < reglen) {
2611 cxgb_get_regs(sc, regs, buf);
2612 error = copyout(buf, regs->data, reglen);
2615 free(buf, M_DEVBUF);
2619 case CHELSIO_SET_HW_SCHED: {
2620 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2621 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2623 if ((sc->flags & FULL_INIT_DONE) == 0)
2624 return (EAGAIN); /* need TP to be initialized */
2625 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2626 !in_range(t->channel, 0, 1) ||
2627 !in_range(t->kbps, 0, 10000000) ||
2628 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2629 !in_range(t->flow_ipg, 0,
2630 dack_ticks_to_usec(sc, 0x7ff)))
2634 error = t3_config_sched(sc, t->kbps, t->sched);
2638 if (t->class_ipg >= 0)
2639 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2640 if (t->flow_ipg >= 0) {
2641 t->flow_ipg *= 1000; /* us -> ns */
2642 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2645 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2647 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2648 bit, t->mode ? bit : 0);
2650 if (t->channel >= 0)
2651 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2652 1 << t->sched, t->channel << t->sched);
2656 return (EOPNOTSUPP);
2663 static __inline void
2664 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2667 uint32_t *p = (uint32_t *)buf + start;
2669 for ( ; start <= end; start += sizeof(uint32_t))
2670 *p++ = t3_read_reg(ap, start);
2673 #define T3_REGMAP_SIZE (3 * 1024)
2675 cxgb_get_regs_len(void)
2677 return T3_REGMAP_SIZE;
2679 #undef T3_REGMAP_SIZE
2682 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf)
2687 * bits 0..9: chip version
2688 * bits 10..15: chip revision
2689 * bit 31: set for PCIe cards
2691 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2694 * We skip the MAC statistics registers because they are clear-on-read.
2695 * Also reading multi-register stats would need to synchronize with the
2696 * periodic mac stats accumulation. Hard to justify the complexity.
2698 memset(buf, 0, REGDUMP_SIZE);
2699 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2700 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2701 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2702 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2703 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2704 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2705 XGM_REG(A_XGM_SERDES_STAT3, 1));
2706 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2707 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2711 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1);