1 /**************************************************************************
2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 Copyright (c) 2007-2009, Chelsio Inc.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
29 ***************************************************************************/
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
40 #include <sys/module.h>
41 #include <sys/pciio.h>
43 #include <machine/bus.h>
44 #include <machine/resource.h>
47 #include <sys/ioccom.h>
49 #include <sys/linker.h>
50 #include <sys/firmware.h>
51 #include <sys/socket.h>
52 #include <sys/sockio.h>
54 #include <sys/sysctl.h>
55 #include <sys/syslog.h>
56 #include <sys/queue.h>
57 #include <sys/taskqueue.h>
61 #include <net/ethernet.h>
63 #include <net/if_var.h>
64 #include <net/if_arp.h>
65 #include <net/if_dl.h>
66 #include <net/if_media.h>
67 #include <net/if_types.h>
68 #include <net/if_vlan_var.h>
70 #include <netinet/in_systm.h>
71 #include <netinet/in.h>
72 #include <netinet/if_ether.h>
73 #include <netinet/ip.h>
74 #include <netinet/ip.h>
75 #include <netinet/tcp.h>
76 #include <netinet/udp.h>
77 #include <netinet/netdump/netdump.h>
79 #include <dev/pci/pcireg.h>
80 #include <dev/pci/pcivar.h>
81 #include <dev/pci/pci_private.h>
83 #include <cxgb_include.h>
89 static int cxgb_setup_interrupts(adapter_t *);
90 static void cxgb_teardown_interrupts(adapter_t *);
91 static void cxgb_init(void *);
92 static int cxgb_init_locked(struct port_info *);
93 static int cxgb_uninit_locked(struct port_info *);
94 static int cxgb_uninit_synchronized(struct port_info *);
95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
96 static int cxgb_media_change(struct ifnet *);
97 static int cxgb_ifm_type(int);
98 static void cxgb_build_medialist(struct port_info *);
99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
100 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
101 static int setup_sge_qsets(adapter_t *);
102 static void cxgb_async_intr(void *);
103 static void cxgb_tick_handler(void *, int);
104 static void cxgb_tick(void *);
105 static void link_check_callout(void *);
106 static void check_link_status(void *, int);
107 static void setup_rss(adapter_t *sc);
108 static int alloc_filters(struct adapter *);
109 static int setup_hw_filters(struct adapter *);
110 static int set_filter(struct adapter *, int, const struct filter_info *);
111 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
112 unsigned int, u64, u64);
113 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
114 unsigned int, u64, u64);
116 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
119 /* Attachment glue for the PCI controller end of the device. Each port of
120 * the device is attached separately, as defined later.
122 static int cxgb_controller_probe(device_t);
123 static int cxgb_controller_attach(device_t);
124 static int cxgb_controller_detach(device_t);
125 static void cxgb_free(struct adapter *);
126 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
128 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
129 static int cxgb_get_regs_len(void);
130 static void touch_bars(device_t dev);
131 static void cxgb_update_mac_settings(struct port_info *p);
133 static int toe_capability(struct port_info *, int);
136 /* Table for probing the cards. The desc field isn't actually used */
142 } cxgb_identifiers[] = {
143 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
144 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
145 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
146 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
147 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
148 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
149 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
150 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
151 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
152 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
153 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
154 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
155 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
156 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
160 static device_method_t cxgb_controller_methods[] = {
161 DEVMETHOD(device_probe, cxgb_controller_probe),
162 DEVMETHOD(device_attach, cxgb_controller_attach),
163 DEVMETHOD(device_detach, cxgb_controller_detach),
168 static driver_t cxgb_controller_driver = {
170 cxgb_controller_methods,
171 sizeof(struct adapter)
174 static int cxgbc_mod_event(module_t, int, void *);
175 static devclass_t cxgb_controller_devclass;
176 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
178 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
179 nitems(cxgb_identifiers) - 1);
180 MODULE_VERSION(cxgbc, 1);
181 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
184 * Attachment glue for the ports. Attachment is done directly to the
187 static int cxgb_port_probe(device_t);
188 static int cxgb_port_attach(device_t);
189 static int cxgb_port_detach(device_t);
191 static device_method_t cxgb_port_methods[] = {
192 DEVMETHOD(device_probe, cxgb_port_probe),
193 DEVMETHOD(device_attach, cxgb_port_attach),
194 DEVMETHOD(device_detach, cxgb_port_detach),
198 static driver_t cxgb_port_driver = {
204 static d_ioctl_t cxgb_extension_ioctl;
205 static d_open_t cxgb_extension_open;
206 static d_close_t cxgb_extension_close;
208 static struct cdevsw cxgb_cdevsw = {
209 .d_version = D_VERSION,
211 .d_open = cxgb_extension_open,
212 .d_close = cxgb_extension_close,
213 .d_ioctl = cxgb_extension_ioctl,
217 static devclass_t cxgb_port_devclass;
218 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
219 MODULE_VERSION(cxgb, 1);
221 NETDUMP_DEFINE(cxgb);
223 static struct mtx t3_list_lock;
224 static SLIST_HEAD(, adapter) t3_list;
226 static struct mtx t3_uld_list_lock;
227 static SLIST_HEAD(, uld_info) t3_uld_list;
231 * The driver uses the best interrupt scheme available on a platform in the
232 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
233 * of these schemes the driver may consider as follows:
235 * msi = 2: choose from among all three options
236 * msi = 1 : only consider MSI and pin interrupts
237 * msi = 0: force pin interrupts
239 static int msi_allowed = 2;
241 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
242 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
243 "MSI-X, MSI, INTx selector");
246 * The driver uses an auto-queue algorithm by default.
247 * To disable it and force a single queue-set per port, use multiq = 0
249 static int multiq = 1;
250 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
251 "use min(ncpus/ports, 8) queue-sets per port");
254 * By default the driver will not update the firmware unless
255 * it was compiled against a newer version
258 static int force_fw_update = 0;
259 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
260 "update firmware even if up to date");
262 int cxgb_use_16k_clusters = -1;
263 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
264 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
266 static int nfilters = -1;
267 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
268 &nfilters, 0, "max number of entries in the filter table");
271 MAX_TXQ_ENTRIES = 16384,
272 MAX_CTRL_TXQ_ENTRIES = 1024,
273 MAX_RSPQ_ENTRIES = 16384,
274 MAX_RX_BUFFERS = 16384,
275 MAX_RX_JUMBO_BUFFERS = 16384,
277 MIN_CTRL_TXQ_ENTRIES = 4,
278 MIN_RSPQ_ENTRIES = 32,
280 MIN_FL_JUMBO_ENTRIES = 32
295 u32 report_filter_id:1;
303 enum { FILTER_NO_VLAN_PRI = 7 };
305 #define EEPROM_MAGIC 0x38E2F10C
307 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
310 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
314 t3rev2char(struct adapter *adapter)
318 switch(adapter->params.rev) {
333 static struct cxgb_ident *
334 cxgb_get_ident(device_t dev)
336 struct cxgb_ident *id;
338 for (id = cxgb_identifiers; id->desc != NULL; id++) {
339 if ((id->vendor == pci_get_vendor(dev)) &&
340 (id->device == pci_get_device(dev))) {
347 static const struct adapter_info *
348 cxgb_get_adapter_info(device_t dev)
350 struct cxgb_ident *id;
351 const struct adapter_info *ai;
353 id = cxgb_get_ident(dev);
357 ai = t3_get_adapter_info(id->index);
363 cxgb_controller_probe(device_t dev)
365 const struct adapter_info *ai;
366 char *ports, buf[80];
369 ai = cxgb_get_adapter_info(dev);
373 nports = ai->nports0 + ai->nports1;
379 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
380 device_set_desc_copy(dev, buf);
381 return (BUS_PROBE_DEFAULT);
384 #define FW_FNAME "cxgb_t3fw"
385 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
386 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
389 upgrade_fw(adapter_t *sc)
391 const struct firmware *fw;
395 if ((fw = firmware_get(FW_FNAME)) == NULL) {
396 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
399 device_printf(sc->dev, "installing firmware on card\n");
400 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
403 device_printf(sc->dev, "failed to install firmware: %d\n",
406 t3_get_fw_version(sc, &vers);
407 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
408 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
409 G_FW_VERSION_MICRO(vers));
412 firmware_put(fw, FIRMWARE_UNLOAD);
418 * The cxgb_controller_attach function is responsible for the initial
419 * bringup of the device. Its responsibilities include:
421 * 1. Determine if the device supports MSI or MSI-X.
422 * 2. Allocate bus resources so that we can access the Base Address Register
423 * 3. Create and initialize mutexes for the controller and its control
424 * logic such as SGE and MDIO.
425 * 4. Call hardware specific setup routine for the adapter as a whole.
426 * 5. Allocate the BAR for doing MSI-X.
427 * 6. Setup the line interrupt iff MSI-X is not supported.
428 * 7. Create the driver's taskq.
429 * 8. Start one task queue service thread.
430 * 9. Check if the firmware and SRAM are up-to-date. They will be
431 * auto-updated later (before FULL_INIT_DONE), if required.
432 * 10. Create a child device for each MAC (port)
433 * 11. Initialize T3 private state.
434 * 12. Trigger the LED
435 * 13. Setup offload iff supported.
436 * 14. Reset/restart the tick callout.
439 * NOTE: Any modification or deviation from this list MUST be reflected in
440 * the above comment. Failure to do so will result in problems on various
441 * error conditions including link flapping.
444 cxgb_controller_attach(device_t dev)
447 const struct adapter_info *ai;
455 sc = device_get_softc(dev);
458 ai = cxgb_get_adapter_info(dev);
460 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
461 device_get_unit(dev));
462 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
464 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
465 device_get_unit(dev));
466 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
467 device_get_unit(dev));
468 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
469 device_get_unit(dev));
471 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
472 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
473 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
475 mtx_lock(&t3_list_lock);
476 SLIST_INSERT_HEAD(&t3_list, sc, link);
477 mtx_unlock(&t3_list_lock);
479 /* find the PCIe link width and set max read request to 4KB*/
480 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) {
483 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
484 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
485 if (sc->link_width < 8 &&
486 (ai->caps & SUPPORTED_10000baseT_Full)) {
487 device_printf(sc->dev,
488 "PCIe x%d Link, expect reduced performance\n",
492 pci_set_max_read_req(dev, 4096);
496 pci_enable_busmaster(dev);
498 * Allocate the registers and make them available to the driver.
499 * The registers that we care about for NIC mode are in BAR 0
501 sc->regs_rid = PCIR_BAR(0);
502 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
503 &sc->regs_rid, RF_ACTIVE)) == NULL) {
504 device_printf(dev, "Cannot allocate BAR region 0\n");
509 sc->bt = rman_get_bustag(sc->regs_res);
510 sc->bh = rman_get_bushandle(sc->regs_res);
511 sc->mmio_len = rman_get_size(sc->regs_res);
513 for (i = 0; i < MAX_NPORTS; i++)
514 sc->port[i].adapter = sc;
516 if (t3_prep_adapter(sc, ai, 1) < 0) {
517 printf("prep adapter failed\n");
522 sc->udbs_rid = PCIR_BAR(2);
524 if (is_offload(sc) &&
525 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
526 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
527 device_printf(dev, "Cannot allocate BAR region 1\n");
532 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
533 * enough messages for the queue sets. If that fails, try falling
534 * back to MSI. If that fails, then try falling back to the legacy
535 * interrupt pin model.
537 sc->msix_regs_rid = 0x20;
538 if ((msi_allowed >= 2) &&
539 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
540 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
543 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
544 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
546 if (pci_msix_count(dev) == 0 ||
547 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
548 sc->msi_count != msi_needed) {
549 device_printf(dev, "alloc msix failed - "
550 "msi_count=%d, msi_needed=%d, err=%d; "
551 "will try MSI\n", sc->msi_count,
555 pci_release_msi(dev);
556 bus_release_resource(dev, SYS_RES_MEMORY,
557 sc->msix_regs_rid, sc->msix_regs_res);
558 sc->msix_regs_res = NULL;
560 sc->flags |= USING_MSIX;
561 sc->cxgb_intr = cxgb_async_intr;
563 "using MSI-X interrupts (%u vectors)\n",
568 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
570 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
571 device_printf(dev, "alloc msi failed - "
572 "err=%d; will try INTx\n", error);
575 pci_release_msi(dev);
577 sc->flags |= USING_MSI;
578 sc->cxgb_intr = t3_intr_msi;
579 device_printf(dev, "using MSI interrupts\n");
582 if (sc->msi_count == 0) {
583 device_printf(dev, "using line interrupts\n");
584 sc->cxgb_intr = t3b_intr;
587 /* Create a private taskqueue thread for handling driver events */
588 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
589 taskqueue_thread_enqueue, &sc->tq);
590 if (sc->tq == NULL) {
591 device_printf(dev, "failed to allocate controller task queue\n");
595 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
596 device_get_nameunit(dev));
597 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
600 /* Create a periodic callout for checking adapter status */
601 callout_init(&sc->cxgb_tick_ch, 1);
603 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
605 * Warn user that a firmware update will be attempted in init.
607 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
608 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
609 sc->flags &= ~FW_UPTODATE;
611 sc->flags |= FW_UPTODATE;
614 if (t3_check_tpsram_version(sc) < 0) {
616 * Warn user that a firmware update will be attempted in init.
618 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
619 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
620 sc->flags &= ~TPS_UPTODATE;
622 sc->flags |= TPS_UPTODATE;
626 * Create a child device for each MAC. The ethernet attachment
627 * will be done in these children.
629 for (i = 0; i < (sc)->params.nports; i++) {
630 struct port_info *pi;
632 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
633 device_printf(dev, "failed to add child port\n");
639 pi->nqsets = port_qsets;
640 pi->first_qset = i*port_qsets;
642 pi->tx_chan = i >= ai->nports0;
643 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
644 sc->rxpkt_map[pi->txpkt_intf] = i;
645 sc->port[i].tx_chan = i >= ai->nports0;
646 sc->portdev[i] = child;
647 device_set_softc(child, pi);
649 if ((error = bus_generic_attach(dev)) != 0)
652 /* initialize sge private state */
653 t3_sge_init_adapter(sc);
657 error = t3_get_fw_version(sc, &vers);
661 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
662 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
663 G_FW_VERSION_MICRO(vers));
665 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
666 ai->desc, is_offload(sc) ? "R" : "",
667 sc->params.vpd.ec, sc->params.vpd.sn);
668 device_set_desc_copy(dev, buf);
670 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
671 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
672 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
674 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
675 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
676 t3_add_attach_sysctls(sc);
679 for (i = 0; i < NUM_CPL_HANDLERS; i++)
680 sc->cpl_handler[i] = cpl_not_handled;
684 error = cxgb_setup_interrupts(sc);
693 * The cxgb_controller_detach routine is called with the device is
694 * unloaded from the system.
698 cxgb_controller_detach(device_t dev)
702 sc = device_get_softc(dev);
710 * The cxgb_free() is called by the cxgb_controller_detach() routine
711 * to tear down the structures that were built up in
712 * cxgb_controller_attach(), and should be the final piece of work
713 * done when fully unloading the driver.
716 * 1. Shutting down the threads started by the cxgb_controller_attach()
718 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
719 * 3. Detaching all of the port devices created during the
720 * cxgb_controller_attach() routine.
721 * 4. Removing the device children created via cxgb_controller_attach().
722 * 5. Releasing PCI resources associated with the device.
723 * 6. Turning off the offload support, iff it was turned on.
724 * 7. Destroying the mutexes created in cxgb_controller_attach().
728 cxgb_free(struct adapter *sc)
733 sc->flags |= CXGB_SHUTDOWN;
737 * Make sure all child devices are gone.
739 bus_generic_detach(sc->dev);
740 for (i = 0; i < (sc)->params.nports; i++) {
741 if (sc->portdev[i] &&
742 device_delete_child(sc->dev, sc->portdev[i]) != 0)
743 device_printf(sc->dev, "failed to delete child port\n");
744 nqsets += sc->port[i].nqsets;
748 * At this point, it is as if cxgb_port_detach has run on all ports, and
749 * cxgb_down has run on the adapter. All interrupts have been silenced,
750 * all open devices have been closed.
752 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
753 __func__, sc->open_device_map));
754 for (i = 0; i < sc->params.nports; i++) {
755 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
760 * Finish off the adapter's callouts.
762 callout_drain(&sc->cxgb_tick_ch);
763 callout_drain(&sc->sge_timer_ch);
766 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
767 * sysctls are cleaned up by the kernel linker.
769 if (sc->flags & FULL_INIT_DONE) {
770 t3_free_sge_resources(sc, nqsets);
771 sc->flags &= ~FULL_INIT_DONE;
775 * Release all interrupt resources.
777 cxgb_teardown_interrupts(sc);
778 if (sc->flags & (USING_MSI | USING_MSIX)) {
779 device_printf(sc->dev, "releasing msi message(s)\n");
780 pci_release_msi(sc->dev);
782 device_printf(sc->dev, "no msi message to release\n");
785 if (sc->msix_regs_res != NULL) {
786 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
791 * Free the adapter's taskqueue.
793 if (sc->tq != NULL) {
794 taskqueue_free(sc->tq);
798 free(sc->filters, M_DEVBUF);
801 if (sc->udbs_res != NULL)
802 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
805 if (sc->regs_res != NULL)
806 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
809 MTX_DESTROY(&sc->mdio_lock);
810 MTX_DESTROY(&sc->sge.reg_lock);
811 MTX_DESTROY(&sc->elmer_lock);
812 mtx_lock(&t3_list_lock);
813 SLIST_REMOVE(&t3_list, sc, adapter, link);
814 mtx_unlock(&t3_list_lock);
815 ADAPTER_LOCK_DEINIT(sc);
819 * setup_sge_qsets - configure SGE Tx/Rx/response queues
820 * @sc: the controller softc
822 * Determines how many sets of SGE queues to use and initializes them.
823 * We support multiple queue sets per port if we have MSI-X, otherwise
824 * just one queue set per port.
827 setup_sge_qsets(adapter_t *sc)
829 int i, j, err, irq_idx = 0, qset_idx = 0;
830 u_int ntxq = SGE_TXQ_PER_SET;
832 if ((err = t3_sge_alloc(sc)) != 0) {
833 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
837 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
840 for (i = 0; i < (sc)->params.nports; i++) {
841 struct port_info *pi = &sc->port[i];
843 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
844 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
845 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
846 &sc->params.sge.qset[qset_idx], ntxq, pi);
848 t3_free_sge_resources(sc, qset_idx);
849 device_printf(sc->dev,
850 "t3_sge_alloc_qset failed with %d\n", err);
860 cxgb_teardown_interrupts(adapter_t *sc)
864 for (i = 0; i < SGE_QSETS; i++) {
865 if (sc->msix_intr_tag[i] == NULL) {
867 /* Should have been setup fully or not at all */
868 KASSERT(sc->msix_irq_res[i] == NULL &&
869 sc->msix_irq_rid[i] == 0,
870 ("%s: half-done interrupt (%d).", __func__, i));
875 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
876 sc->msix_intr_tag[i]);
877 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
878 sc->msix_irq_res[i]);
880 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
881 sc->msix_irq_rid[i] = 0;
885 KASSERT(sc->irq_res != NULL,
886 ("%s: half-done interrupt.", __func__));
888 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
889 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
892 sc->irq_res = sc->intr_tag = NULL;
898 cxgb_setup_interrupts(adapter_t *sc)
900 struct resource *res;
902 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
904 sc->irq_rid = intr_flag ? 1 : 0;
905 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
906 RF_SHAREABLE | RF_ACTIVE);
907 if (sc->irq_res == NULL) {
908 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
909 intr_flag, sc->irq_rid);
913 err = bus_setup_intr(sc->dev, sc->irq_res,
914 INTR_MPSAFE | INTR_TYPE_NET, NULL,
915 sc->cxgb_intr, sc, &sc->intr_tag);
918 device_printf(sc->dev,
919 "Cannot set up interrupt (%x, %u, %d)\n",
920 intr_flag, sc->irq_rid, err);
921 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
923 sc->irq_res = sc->intr_tag = NULL;
928 /* That's all for INTx or MSI */
929 if (!(intr_flag & USING_MSIX) || err)
932 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
933 for (i = 0; i < sc->msi_count - 1; i++) {
935 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
936 RF_SHAREABLE | RF_ACTIVE);
938 device_printf(sc->dev, "Cannot allocate interrupt "
939 "for message %d\n", rid);
944 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
945 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
947 device_printf(sc->dev, "Cannot set up interrupt "
948 "for message %d (%d)\n", rid, err);
949 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
953 sc->msix_irq_rid[i] = rid;
954 sc->msix_irq_res[i] = res;
955 sc->msix_intr_tag[i] = tag;
956 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
960 cxgb_teardown_interrupts(sc);
967 cxgb_port_probe(device_t dev)
973 p = device_get_softc(dev);
975 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
976 device_set_desc_copy(dev, buf);
982 cxgb_makedev(struct port_info *pi)
985 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
986 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
988 if (pi->port_cdev == NULL)
991 pi->port_cdev->si_drv1 = (void *)pi;
996 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
997 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
998 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
999 #define CXGB_CAP_ENABLE CXGB_CAP
1002 cxgb_port_attach(device_t dev)
1004 struct port_info *p;
1009 p = device_get_softc(dev);
1011 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1012 device_get_unit(device_get_parent(dev)), p->port_id);
1013 PORT_LOCK_INIT(p, p->lockbuf);
1015 callout_init(&p->link_check_ch, 1);
1016 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1018 /* Allocate an ifnet object and set it up */
1019 ifp = p->ifp = if_alloc(IFT_ETHER);
1021 device_printf(dev, "Cannot allocate ifnet\n");
1025 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1026 ifp->if_init = cxgb_init;
1028 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1029 ifp->if_ioctl = cxgb_ioctl;
1030 ifp->if_transmit = cxgb_transmit;
1031 ifp->if_qflush = cxgb_qflush;
1032 ifp->if_get_counter = cxgb_get_counter;
1034 ifp->if_capabilities = CXGB_CAP;
1037 ifp->if_capabilities |= IFCAP_TOE4;
1039 ifp->if_capenable = CXGB_CAP_ENABLE;
1040 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1041 CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1044 * Disable TSO on 4-port - it isn't supported by the firmware.
1046 if (sc->params.nports > 2) {
1047 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1048 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1049 ifp->if_hwassist &= ~CSUM_TSO;
1052 ether_ifattach(ifp, p->hw_addr);
1054 /* Attach driver netdump methods. */
1055 NETDUMP_SET(ifp, cxgb);
1057 #ifdef DEFAULT_JUMBO
1058 if (sc->params.nports <= 2)
1059 ifp->if_mtu = ETHERMTU_JUMBO;
1061 if ((err = cxgb_makedev(p)) != 0) {
1062 printf("makedev failed %d\n", err);
1066 /* Create a list of media supported by this port */
1067 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1069 cxgb_build_medialist(p);
1071 t3_sge_init_port(p);
1077 * cxgb_port_detach() is called via the device_detach methods when
1078 * cxgb_free() calls the bus_generic_detach. It is responsible for
1079 * removing the device from the view of the kernel, i.e. from all
1080 * interfaces lists etc. This routine is only called when the driver is
1081 * being unloaded, not when the link goes down.
1084 cxgb_port_detach(device_t dev)
1086 struct port_info *p;
1090 p = device_get_softc(dev);
1093 /* Tell cxgb_ioctl and if_init that the port is going away */
1098 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1102 if (p->port_cdev != NULL)
1103 destroy_dev(p->port_cdev);
1105 cxgb_uninit_synchronized(p);
1106 ether_ifdetach(p->ifp);
1108 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1109 struct sge_qset *qs = &sc->sge.qs[i];
1110 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1112 callout_drain(&txq->txq_watchdog);
1113 callout_drain(&txq->txq_timer);
1116 PORT_LOCK_DEINIT(p);
1122 wakeup_one(&sc->flags);
1128 t3_fatal_err(struct adapter *sc)
1132 if (sc->flags & FULL_INIT_DONE) {
1134 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1135 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1136 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1137 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1138 t3_intr_disable(sc);
1140 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1141 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1142 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1143 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1147 t3_os_find_pci_capability(adapter_t *sc, int cap)
1150 struct pci_devinfo *dinfo;
1156 dinfo = device_get_ivars(dev);
1159 status = pci_read_config(dev, PCIR_STATUS, 2);
1160 if (!(status & PCIM_STATUS_CAPPRESENT))
1163 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1169 ptr = PCIR_CAP_PTR_2;
1175 ptr = pci_read_config(dev, ptr, 1);
1178 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1180 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1187 t3_os_pci_save_state(struct adapter *sc)
1190 struct pci_devinfo *dinfo;
1193 dinfo = device_get_ivars(dev);
1195 pci_cfg_save(dev, dinfo, 0);
1200 t3_os_pci_restore_state(struct adapter *sc)
1203 struct pci_devinfo *dinfo;
1206 dinfo = device_get_ivars(dev);
1208 pci_cfg_restore(dev, dinfo);
1213 * t3_os_link_changed - handle link status changes
1214 * @sc: the adapter associated with the link change
1215 * @port_id: the port index whose link status has changed
1216 * @link_status: the new status of the link
1217 * @speed: the new speed setting
1218 * @duplex: the new duplex setting
1219 * @fc: the new flow-control setting
1221 * This is the OS-dependent handler for link status changes. The OS
1222 * neutral handler takes care of most of the processing for these events,
1223 * then calls this handler for any OS-specific processing.
1226 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1227 int duplex, int fc, int mac_was_reset)
1229 struct port_info *pi = &adapter->port[port_id];
1230 struct ifnet *ifp = pi->ifp;
1232 /* no race with detach, so ifp should always be good */
1233 KASSERT(ifp, ("%s: if detached.", __func__));
1235 /* Reapply mac settings if they were lost due to a reset */
1236 if (mac_was_reset) {
1238 cxgb_update_mac_settings(pi);
1243 ifp->if_baudrate = IF_Mbps(speed);
1244 if_link_state_change(ifp, LINK_STATE_UP);
1246 if_link_state_change(ifp, LINK_STATE_DOWN);
1250 * t3_os_phymod_changed - handle PHY module changes
1251 * @phy: the PHY reporting the module change
1252 * @mod_type: new module type
1254 * This is the OS-dependent handler for PHY module changes. It is
1255 * invoked when a PHY module is removed or inserted for any OS-specific
1258 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1260 static const char *mod_str[] = {
1261 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1263 struct port_info *pi = &adap->port[port_id];
1264 int mod = pi->phy.modtype;
1266 if (mod != pi->media.ifm_cur->ifm_data)
1267 cxgb_build_medialist(pi);
1269 if (mod == phy_modtype_none)
1270 if_printf(pi->ifp, "PHY module unplugged\n");
1272 KASSERT(mod < ARRAY_SIZE(mod_str),
1273 ("invalid PHY module type %d", mod));
1274 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1279 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1283 * The ifnet might not be allocated before this gets called,
1284 * as this is called early on in attach by t3_prep_adapter
1285 * save the address off in the port structure
1288 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1289 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1293 * Programs the XGMAC based on the settings in the ifnet. These settings
1294 * include MTU, MAC address, mcast addresses, etc.
1297 cxgb_update_mac_settings(struct port_info *p)
1299 struct ifnet *ifp = p->ifp;
1300 struct t3_rx_mode rm;
1301 struct cmac *mac = &p->mac;
1304 PORT_LOCK_ASSERT_OWNED(p);
1306 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1309 if (ifp->if_capenable & IFCAP_VLAN_MTU)
1310 mtu += ETHER_VLAN_ENCAP_LEN;
1312 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1314 t3_mac_set_mtu(mac, mtu);
1315 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1316 t3_mac_set_address(mac, 0, p->hw_addr);
1317 t3_init_rx_mode(&rm, p);
1318 t3_mac_set_rx_mode(mac, &rm);
1323 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1328 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1337 init_tp_parity(struct adapter *adap)
1341 struct cpl_set_tcb_field *greq;
1342 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1344 t3_tp_set_offload_mode(adap, 1);
1346 for (i = 0; i < 16; i++) {
1347 struct cpl_smt_write_req *req;
1349 m = m_gethdr(M_WAITOK, MT_DATA);
1350 req = mtod(m, struct cpl_smt_write_req *);
1351 m->m_len = m->m_pkthdr.len = sizeof(*req);
1352 memset(req, 0, sizeof(*req));
1353 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1354 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1356 t3_mgmt_tx(adap, m);
1359 for (i = 0; i < 2048; i++) {
1360 struct cpl_l2t_write_req *req;
1362 m = m_gethdr(M_WAITOK, MT_DATA);
1363 req = mtod(m, struct cpl_l2t_write_req *);
1364 m->m_len = m->m_pkthdr.len = sizeof(*req);
1365 memset(req, 0, sizeof(*req));
1366 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1367 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1368 req->params = htonl(V_L2T_W_IDX(i));
1369 t3_mgmt_tx(adap, m);
1372 for (i = 0; i < 2048; i++) {
1373 struct cpl_rte_write_req *req;
1375 m = m_gethdr(M_WAITOK, MT_DATA);
1376 req = mtod(m, struct cpl_rte_write_req *);
1377 m->m_len = m->m_pkthdr.len = sizeof(*req);
1378 memset(req, 0, sizeof(*req));
1379 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1380 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1381 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1382 t3_mgmt_tx(adap, m);
1385 m = m_gethdr(M_WAITOK, MT_DATA);
1386 greq = mtod(m, struct cpl_set_tcb_field *);
1387 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1388 memset(greq, 0, sizeof(*greq));
1389 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1390 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1391 greq->mask = htobe64(1);
1392 t3_mgmt_tx(adap, m);
1394 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1395 t3_tp_set_offload_mode(adap, 0);
1400 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1401 * @adap: the adapter
1403 * Sets up RSS to distribute packets to multiple receive queues. We
1404 * configure the RSS CPU lookup table to distribute to the number of HW
1405 * receive queues, and the response queue lookup table to narrow that
1406 * down to the response queues actually configured for each port.
1407 * We always configure the RSS mapping for two ports since the mapping
1408 * table has plenty of entries.
1411 setup_rss(adapter_t *adap)
1415 uint8_t cpus[SGE_QSETS + 1];
1416 uint16_t rspq_map[RSS_TABLE_SIZE];
1418 for (i = 0; i < SGE_QSETS; ++i)
1420 cpus[SGE_QSETS] = 0xff;
1423 for_each_port(adap, i) {
1424 const struct port_info *pi = adap2pinfo(adap, i);
1426 nq[pi->tx_chan] += pi->nqsets;
1428 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1429 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1430 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1433 /* Calculate the reverse RSS map table */
1434 for (i = 0; i < SGE_QSETS; ++i)
1435 adap->rrss_map[i] = 0xff;
1436 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1437 if (adap->rrss_map[rspq_map[i]] == 0xff)
1438 adap->rrss_map[rspq_map[i]] = i;
1440 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1441 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1442 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1447 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1451 struct mngt_pktsched_wr *req;
1453 m = m_gethdr(M_NOWAIT, MT_DATA);
1455 req = mtod(m, struct mngt_pktsched_wr *);
1456 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1457 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1462 req->binding = port;
1463 m->m_len = m->m_pkthdr.len = sizeof(*req);
1464 t3_mgmt_tx(adap, m);
1469 bind_qsets(adapter_t *sc)
1473 for (i = 0; i < (sc)->params.nports; ++i) {
1474 const struct port_info *pi = adap2pinfo(sc, i);
1476 for (j = 0; j < pi->nqsets; ++j) {
1477 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1485 update_tpeeprom(struct adapter *adap)
1487 const struct firmware *tpeeprom;
1490 unsigned int major, minor;
1494 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1496 major = G_TP_VERSION_MAJOR(version);
1497 minor = G_TP_VERSION_MINOR(version);
1498 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1501 rev = t3rev2char(adap);
1502 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1504 tpeeprom = firmware_get(name);
1505 if (tpeeprom == NULL) {
1506 device_printf(adap->dev,
1507 "could not load TP EEPROM: unable to load %s\n",
1512 len = tpeeprom->datasize - 4;
1514 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1516 goto release_tpeeprom;
1518 if (len != TP_SRAM_LEN) {
1519 device_printf(adap->dev,
1520 "%s length is wrong len=%d expected=%d\n", name,
1525 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1529 device_printf(adap->dev,
1530 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1531 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1533 device_printf(adap->dev,
1534 "Protocol SRAM image update in EEPROM failed\n");
1537 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1543 update_tpsram(struct adapter *adap)
1545 const struct firmware *tpsram;
1549 rev = t3rev2char(adap);
1550 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1552 update_tpeeprom(adap);
1554 tpsram = firmware_get(name);
1555 if (tpsram == NULL){
1556 device_printf(adap->dev, "could not load TP SRAM\n");
1559 device_printf(adap->dev, "updating TP SRAM\n");
1561 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1563 goto release_tpsram;
1565 ret = t3_set_proto_sram(adap, tpsram->data);
1567 device_printf(adap->dev, "loading protocol SRAM failed\n");
1570 firmware_put(tpsram, FIRMWARE_UNLOAD);
1576 * cxgb_up - enable the adapter
1577 * @adap: adapter being enabled
1579 * Called when the first port is enabled, this function performs the
1580 * actions necessary to make an adapter operational, such as completing
1581 * the initialization of HW modules, and enabling interrupts.
1584 cxgb_up(struct adapter *sc)
1587 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1589 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1590 __func__, sc->open_device_map));
1592 if ((sc->flags & FULL_INIT_DONE) == 0) {
1594 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1596 if ((sc->flags & FW_UPTODATE) == 0)
1597 if ((err = upgrade_fw(sc)))
1600 if ((sc->flags & TPS_UPTODATE) == 0)
1601 if ((err = update_tpsram(sc)))
1604 if (is_offload(sc) && nfilters != 0) {
1605 sc->params.mc5.nservers = 0;
1608 sc->params.mc5.nfilters = mxf;
1610 sc->params.mc5.nfilters = min(nfilters, mxf);
1613 err = t3_init_hw(sc, 0);
1617 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1618 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1620 err = setup_sge_qsets(sc);
1627 t3_add_configured_sysctls(sc);
1628 sc->flags |= FULL_INIT_DONE;
1635 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1636 is_offload(sc) && init_tp_parity(sc) == 0)
1637 sc->flags |= TP_PARITY_INIT;
1639 if (sc->flags & TP_PARITY_INIT) {
1640 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1641 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1644 if (!(sc->flags & QUEUES_BOUND)) {
1646 setup_hw_filters(sc);
1647 sc->flags |= QUEUES_BOUND;
1650 t3_sge_reset_adapter(sc);
1656 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1657 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1658 * during controller_detach, not here.
1661 cxgb_down(struct adapter *sc)
1664 t3_intr_disable(sc);
1668 * if_init for cxgb ports.
1671 cxgb_init(void *arg)
1673 struct port_info *p = arg;
1674 struct adapter *sc = p->adapter;
1677 cxgb_init_locked(p); /* releases adapter lock */
1678 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1682 cxgb_init_locked(struct port_info *p)
1684 struct adapter *sc = p->adapter;
1685 struct ifnet *ifp = p->ifp;
1686 struct cmac *mac = &p->mac;
1687 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1689 ADAPTER_LOCK_ASSERT_OWNED(sc);
1691 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1693 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1702 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1705 * The code that runs during one-time adapter initialization can sleep
1706 * so it's important not to hold any locks across it.
1708 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1716 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1720 if (isset(&sc->open_device_map, p->port_id) &&
1721 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1725 t3_port_intr_enable(sc, p->port_id);
1726 if (!mac->multiport)
1728 cxgb_update_mac_settings(p);
1729 t3_link_start(&p->phy, mac, &p->link_config);
1730 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1731 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1732 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1735 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1736 struct sge_qset *qs = &sc->sge.qs[i];
1737 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1739 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1740 txq->txq_watchdog.c_cpu);
1744 setbit(&sc->open_device_map, p->port_id);
1745 callout_reset(&p->link_check_ch,
1746 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1747 link_check_callout, p);
1752 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1756 wakeup_one(&sc->flags);
1762 cxgb_uninit_locked(struct port_info *p)
1764 struct adapter *sc = p->adapter;
1767 ADAPTER_LOCK_ASSERT_OWNED(sc);
1769 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1770 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1779 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1783 rc = cxgb_uninit_synchronized(p);
1786 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1788 wakeup_one(&sc->flags);
1795 * Called on "ifconfig down", and from port_detach
1798 cxgb_uninit_synchronized(struct port_info *pi)
1800 struct adapter *sc = pi->adapter;
1801 struct ifnet *ifp = pi->ifp;
1804 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1806 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1809 * Clear this port's bit from the open device map, and then drain all
1810 * the tasks that can access/manipulate this port's port_info or ifp.
1811 * We disable this port's interrupts here and so the slow/ext
1812 * interrupt tasks won't be enqueued. The tick task will continue to
1813 * be enqueued every second but the runs after this drain will not see
1814 * this port in the open device map.
1816 * A well behaved task must take open_device_map into account and ignore
1817 * ports that are not open.
1819 clrbit(&sc->open_device_map, pi->port_id);
1820 t3_port_intr_disable(sc, pi->port_id);
1821 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1822 taskqueue_drain(sc->tq, &sc->tick_task);
1824 callout_drain(&pi->link_check_ch);
1825 taskqueue_drain(sc->tq, &pi->link_check_task);
1828 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1830 /* disable pause frames */
1831 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1833 /* Reset RX FIFO HWM */
1834 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1835 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1839 /* Wait for TXFIFO empty */
1840 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1841 F_TXFIFO_EMPTY, 1, 20, 5);
1844 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1846 pi->phy.ops->power_down(&pi->phy, 1);
1850 pi->link_config.link_ok = 0;
1851 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1853 if (sc->open_device_map == 0)
1854 cxgb_down(pi->adapter);
1860 * Mark lro enabled or disabled in all qsets for this port
1863 cxgb_set_lro(struct port_info *p, int enabled)
1866 struct adapter *adp = p->adapter;
1869 for (i = 0; i < p->nqsets; i++) {
1870 q = &adp->sge.qs[p->first_qset + i];
1871 q->lro.enabled = (enabled != 0);
1877 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1879 struct port_info *p = ifp->if_softc;
1880 struct adapter *sc = p->adapter;
1881 struct ifreq *ifr = (struct ifreq *)data;
1882 int flags, error = 0, mtu;
1888 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1896 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1901 cxgb_update_mac_settings(p);
1912 if (ifp->if_flags & IFF_UP) {
1913 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1914 flags = p->if_flags;
1915 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1916 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
1922 cxgb_update_mac_settings(p);
1927 error = cxgb_init_locked(p);
1928 p->if_flags = ifp->if_flags;
1929 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1930 error = cxgb_uninit_locked(p);
1934 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1939 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1943 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1945 cxgb_update_mac_settings(p);
1953 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1957 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1958 if (mask & IFCAP_TXCSUM) {
1959 ifp->if_capenable ^= IFCAP_TXCSUM;
1960 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1962 if (IFCAP_TSO4 & ifp->if_capenable &&
1963 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1964 ifp->if_capenable &= ~IFCAP_TSO4;
1966 "tso4 disabled due to -txcsum.\n");
1969 if (mask & IFCAP_TXCSUM_IPV6) {
1970 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1971 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1973 if (IFCAP_TSO6 & ifp->if_capenable &&
1974 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1975 ifp->if_capenable &= ~IFCAP_TSO6;
1977 "tso6 disabled due to -txcsum6.\n");
1980 if (mask & IFCAP_RXCSUM)
1981 ifp->if_capenable ^= IFCAP_RXCSUM;
1982 if (mask & IFCAP_RXCSUM_IPV6)
1983 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1986 * Note that we leave CSUM_TSO alone (it is always set). The
1987 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1988 * sending a TSO request our way, so it's sufficient to toggle
1991 if (mask & IFCAP_TSO4) {
1992 if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1993 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1994 if_printf(ifp, "enable txcsum first.\n");
1998 ifp->if_capenable ^= IFCAP_TSO4;
2000 if (mask & IFCAP_TSO6) {
2001 if (!(IFCAP_TSO6 & ifp->if_capenable) &&
2002 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2003 if_printf(ifp, "enable txcsum6 first.\n");
2007 ifp->if_capenable ^= IFCAP_TSO6;
2009 if (mask & IFCAP_LRO) {
2010 ifp->if_capenable ^= IFCAP_LRO;
2012 /* Safe to do this even if cxgb_up not called yet */
2013 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2016 if (mask & IFCAP_TOE4) {
2017 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
2019 error = toe_capability(p, enable);
2021 ifp->if_capenable ^= mask;
2024 if (mask & IFCAP_VLAN_HWTAGGING) {
2025 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2026 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2028 cxgb_update_mac_settings(p);
2032 if (mask & IFCAP_VLAN_MTU) {
2033 ifp->if_capenable ^= IFCAP_VLAN_MTU;
2034 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2036 cxgb_update_mac_settings(p);
2040 if (mask & IFCAP_VLAN_HWTSO)
2041 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2042 if (mask & IFCAP_VLAN_HWCSUM)
2043 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2045 #ifdef VLAN_CAPABILITIES
2046 VLAN_CAPABILITIES(ifp);
2052 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2055 error = ether_ioctl(ifp, command, data);
2062 cxgb_media_change(struct ifnet *ifp)
2064 return (EOPNOTSUPP);
2068 * Translates phy->modtype to the correct Ethernet media subtype.
2071 cxgb_ifm_type(int mod)
2074 case phy_modtype_sr:
2075 return (IFM_10G_SR);
2076 case phy_modtype_lr:
2077 return (IFM_10G_LR);
2078 case phy_modtype_lrm:
2079 return (IFM_10G_LRM);
2080 case phy_modtype_twinax:
2081 return (IFM_10G_TWINAX);
2082 case phy_modtype_twinax_long:
2083 return (IFM_10G_TWINAX_LONG);
2084 case phy_modtype_none:
2086 case phy_modtype_unknown:
2087 return (IFM_UNKNOWN);
2090 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2091 return (IFM_UNKNOWN);
2095 * Rebuilds the ifmedia list for this port, and sets the current media.
2098 cxgb_build_medialist(struct port_info *p)
2100 struct cphy *phy = &p->phy;
2101 struct ifmedia *media = &p->media;
2102 int mod = phy->modtype;
2103 int m = IFM_ETHER | IFM_FDX;
2107 ifmedia_removeall(media);
2108 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2111 if (phy->caps & SUPPORTED_10000baseT_Full)
2112 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2114 if (phy->caps & SUPPORTED_1000baseT_Full)
2115 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2117 if (phy->caps & SUPPORTED_100baseT_Full)
2118 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2120 if (phy->caps & SUPPORTED_10baseT_Full)
2121 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2123 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2124 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2126 } else if (phy->caps & SUPPORTED_TP) {
2129 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2130 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2132 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2133 ifmedia_set(media, m | IFM_10G_CX4);
2135 } else if (phy->caps & SUPPORTED_FIBRE &&
2136 phy->caps & SUPPORTED_10000baseT_Full) {
2137 /* 10G optical (but includes SFP+ twinax) */
2139 m |= cxgb_ifm_type(mod);
2140 if (IFM_SUBTYPE(m) == IFM_NONE)
2143 ifmedia_add(media, m, mod, NULL);
2144 ifmedia_set(media, m);
2146 } else if (phy->caps & SUPPORTED_FIBRE &&
2147 phy->caps & SUPPORTED_1000baseT_Full) {
2150 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2151 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2152 ifmedia_set(media, m | IFM_1000_SX);
2155 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2163 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2165 struct port_info *p = ifp->if_softc;
2166 struct ifmedia_entry *cur = p->media.ifm_cur;
2167 int speed = p->link_config.speed;
2169 if (cur->ifm_data != p->phy.modtype) {
2170 cxgb_build_medialist(p);
2171 cur = p->media.ifm_cur;
2174 ifmr->ifm_status = IFM_AVALID;
2175 if (!p->link_config.link_ok)
2178 ifmr->ifm_status |= IFM_ACTIVE;
2181 * active and current will differ iff current media is autoselect. That
2182 * can happen only for copper RJ45.
2184 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2186 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2187 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2189 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2190 if (speed == SPEED_10000)
2191 ifmr->ifm_active |= IFM_10G_T;
2192 else if (speed == SPEED_1000)
2193 ifmr->ifm_active |= IFM_1000_T;
2194 else if (speed == SPEED_100)
2195 ifmr->ifm_active |= IFM_100_TX;
2196 else if (speed == SPEED_10)
2197 ifmr->ifm_active |= IFM_10_T;
2199 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2204 cxgb_get_counter(struct ifnet *ifp, ift_counter c)
2206 struct port_info *pi = ifp->if_softc;
2207 struct adapter *sc = pi->adapter;
2208 struct cmac *mac = &pi->mac;
2209 struct mac_stats *mstats = &mac->stats;
2211 cxgb_refresh_stats(pi);
2214 case IFCOUNTER_IPACKETS:
2215 return (mstats->rx_frames);
2217 case IFCOUNTER_IERRORS:
2218 return (mstats->rx_jabber + mstats->rx_data_errs +
2219 mstats->rx_sequence_errs + mstats->rx_runt +
2220 mstats->rx_too_long + mstats->rx_mac_internal_errs +
2221 mstats->rx_short + mstats->rx_fcs_errs);
2223 case IFCOUNTER_OPACKETS:
2224 return (mstats->tx_frames);
2226 case IFCOUNTER_OERRORS:
2227 return (mstats->tx_excess_collisions + mstats->tx_underrun +
2228 mstats->tx_len_errs + mstats->tx_mac_internal_errs +
2229 mstats->tx_excess_deferral + mstats->tx_fcs_errs);
2231 case IFCOUNTER_COLLISIONS:
2232 return (mstats->tx_total_collisions);
2234 case IFCOUNTER_IBYTES:
2235 return (mstats->rx_octets);
2237 case IFCOUNTER_OBYTES:
2238 return (mstats->tx_octets);
2240 case IFCOUNTER_IMCASTS:
2241 return (mstats->rx_mcast_frames);
2243 case IFCOUNTER_OMCASTS:
2244 return (mstats->tx_mcast_frames);
2246 case IFCOUNTER_IQDROPS:
2247 return (mstats->rx_cong_drops);
2249 case IFCOUNTER_OQDROPS: {
2254 if (sc->flags & FULL_INIT_DONE) {
2255 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
2256 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
2264 return (if_get_counter_default(ifp, c));
2269 cxgb_async_intr(void *data)
2271 adapter_t *sc = data;
2273 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2274 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2275 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2279 link_check_callout(void *arg)
2281 struct port_info *pi = arg;
2282 struct adapter *sc = pi->adapter;
2284 if (!isset(&sc->open_device_map, pi->port_id))
2287 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2291 check_link_status(void *arg, int pending)
2293 struct port_info *pi = arg;
2294 struct adapter *sc = pi->adapter;
2296 if (!isset(&sc->open_device_map, pi->port_id))
2299 t3_link_changed(sc, pi->port_id);
2301 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
2302 pi->link_config.link_ok == 0)
2303 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2307 t3_os_link_intr(struct port_info *pi)
2310 * Schedule a link check in the near future. If the link is flapping
2311 * rapidly we'll keep resetting the callout and delaying the check until
2312 * things stabilize a bit.
2314 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2318 check_t3b2_mac(struct adapter *sc)
2322 if (sc->flags & CXGB_SHUTDOWN)
2325 for_each_port(sc, i) {
2326 struct port_info *p = &sc->port[i];
2329 struct ifnet *ifp = p->ifp;
2332 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2333 !p->link_config.link_ok)
2336 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2337 ("%s: state mismatch (drv_flags %x, device_map %x)",
2338 __func__, ifp->if_drv_flags, sc->open_device_map));
2341 status = t3b2_mac_watchdog_task(&p->mac);
2343 p->mac.stats.num_toggled++;
2344 else if (status == 2) {
2345 struct cmac *mac = &p->mac;
2347 cxgb_update_mac_settings(p);
2348 t3_link_start(&p->phy, mac, &p->link_config);
2349 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2350 t3_port_intr_enable(sc, p->port_id);
2351 p->mac.stats.num_resets++;
2358 cxgb_tick(void *arg)
2360 adapter_t *sc = (adapter_t *)arg;
2362 if (sc->flags & CXGB_SHUTDOWN)
2365 taskqueue_enqueue(sc->tq, &sc->tick_task);
2366 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2370 cxgb_refresh_stats(struct port_info *pi)
2373 const struct timeval interval = {0, 250000}; /* 250ms */
2376 timevalsub(&tv, &interval);
2377 if (timevalcmp(&tv, &pi->last_refreshed, <))
2381 t3_mac_update_stats(&pi->mac);
2383 getmicrotime(&pi->last_refreshed);
2387 cxgb_tick_handler(void *arg, int count)
2389 adapter_t *sc = (adapter_t *)arg;
2390 const struct adapter_params *p = &sc->params;
2392 uint32_t cause, reset;
2394 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2397 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2400 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2402 struct sge_qset *qs = &sc->sge.qs[0];
2405 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2408 for (i = 0; i < SGE_QSETS; i++) {
2410 qs[i].rspq.starved++;
2414 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2416 for (i = 0; i < SGE_QSETS * 2; i++) {
2418 qs[i / 2].fl[i % 2].empty++;
2424 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2425 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2428 for (i = 0; i < sc->params.nports; i++) {
2429 struct port_info *pi = &sc->port[i];
2430 struct cmac *mac = &pi->mac;
2432 if (!isset(&sc->open_device_map, pi->port_id))
2435 cxgb_refresh_stats(pi);
2440 /* Count rx fifo overflows, once per second */
2441 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2443 if (cause & F_RXFIFO_OVERFLOW) {
2444 mac->stats.rx_fifo_ovfl++;
2445 reset |= F_RXFIFO_OVERFLOW;
2447 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2452 touch_bars(device_t dev)
2457 #if !defined(__LP64__) && 0
2460 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2461 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2462 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2463 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2464 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2465 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2470 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2474 u32 aligned_offset, aligned_len, *p;
2475 struct adapter *adapter = pi->adapter;
2478 aligned_offset = offset & ~3;
2479 aligned_len = (len + (offset & 3) + 3) & ~3;
2481 if (aligned_offset != offset || aligned_len != len) {
2482 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2485 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2486 if (!err && aligned_len > 4)
2487 err = t3_seeprom_read(adapter,
2488 aligned_offset + aligned_len - 4,
2489 (u32 *)&buf[aligned_len - 4]);
2492 memcpy(buf + (offset & 3), data, len);
2494 buf = (uint8_t *)(uintptr_t)data;
2496 err = t3_seeprom_wp(adapter, 0);
2500 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2501 err = t3_seeprom_write(adapter, aligned_offset, *p);
2502 aligned_offset += 4;
2506 err = t3_seeprom_wp(adapter, 1);
2509 free(buf, M_DEVBUF);
2515 in_range(int val, int lo, int hi)
2517 return val < 0 || (val <= hi && val >= lo);
2521 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2527 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2533 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2534 int fflag, struct thread *td)
2537 struct port_info *pi = dev->si_drv1;
2538 adapter_t *sc = pi->adapter;
2540 #ifdef PRIV_SUPPORTED
2541 if (priv_check(td, PRIV_DRIVER)) {
2543 printf("user does not have access to privileged ioctls\n");
2549 printf("user does not have access to privileged ioctls\n");
2555 case CHELSIO_GET_MIIREG: {
2557 struct cphy *phy = &pi->phy;
2558 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2560 if (!phy->mdio_read)
2561 return (EOPNOTSUPP);
2563 mmd = mid->phy_id >> 8;
2566 else if (mmd > MDIO_DEV_VEND2)
2569 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2570 mid->reg_num, &val);
2572 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2573 mid->reg_num & 0x1f, &val);
2578 case CHELSIO_SET_MIIREG: {
2579 struct cphy *phy = &pi->phy;
2580 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2582 if (!phy->mdio_write)
2583 return (EOPNOTSUPP);
2585 mmd = mid->phy_id >> 8;
2588 else if (mmd > MDIO_DEV_VEND2)
2591 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2592 mmd, mid->reg_num, mid->val_in);
2594 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2595 mid->reg_num & 0x1f,
2599 case CHELSIO_SETREG: {
2600 struct ch_reg *edata = (struct ch_reg *)data;
2601 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2603 t3_write_reg(sc, edata->addr, edata->val);
2606 case CHELSIO_GETREG: {
2607 struct ch_reg *edata = (struct ch_reg *)data;
2608 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2610 edata->val = t3_read_reg(sc, edata->addr);
2613 case CHELSIO_GET_SGE_CONTEXT: {
2614 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2615 mtx_lock_spin(&sc->sge.reg_lock);
2616 switch (ecntxt->cntxt_type) {
2617 case CNTXT_TYPE_EGRESS:
2618 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2622 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2625 case CNTXT_TYPE_RSP:
2626 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2630 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2637 mtx_unlock_spin(&sc->sge.reg_lock);
2640 case CHELSIO_GET_SGE_DESC: {
2641 struct ch_desc *edesc = (struct ch_desc *)data;
2643 if (edesc->queue_num >= SGE_QSETS * 6)
2645 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2646 edesc->queue_num % 6, edesc->idx, edesc->data);
2652 case CHELSIO_GET_QSET_PARAMS: {
2653 struct qset_params *q;
2654 struct ch_qset_params *t = (struct ch_qset_params *)data;
2655 int q1 = pi->first_qset;
2656 int nqsets = pi->nqsets;
2659 if (t->qset_idx >= nqsets)
2662 i = q1 + t->qset_idx;
2663 q = &sc->params.sge.qset[i];
2664 t->rspq_size = q->rspq_size;
2665 t->txq_size[0] = q->txq_size[0];
2666 t->txq_size[1] = q->txq_size[1];
2667 t->txq_size[2] = q->txq_size[2];
2668 t->fl_size[0] = q->fl_size;
2669 t->fl_size[1] = q->jumbo_size;
2670 t->polling = q->polling;
2672 t->intr_lat = q->coalesce_usecs;
2673 t->cong_thres = q->cong_thres;
2676 if ((sc->flags & FULL_INIT_DONE) == 0)
2678 else if (sc->flags & USING_MSIX)
2679 t->vector = rman_get_start(sc->msix_irq_res[i]);
2681 t->vector = rman_get_start(sc->irq_res);
2685 case CHELSIO_GET_QSET_NUM: {
2686 struct ch_reg *edata = (struct ch_reg *)data;
2687 edata->val = pi->nqsets;
2690 case CHELSIO_LOAD_FW: {
2693 struct ch_mem_range *t = (struct ch_mem_range *)data;
2696 * You're allowed to load a firmware only before FULL_INIT_DONE
2698 * FW_UPTODATE is also set so the rest of the initialization
2699 * will not overwrite what was loaded here. This gives you the
2700 * flexibility to load any firmware (and maybe shoot yourself in
2705 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2710 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2714 error = copyin(t->buf, fw_data, t->len);
2717 error = -t3_load_fw(sc, fw_data, t->len);
2719 if (t3_get_fw_version(sc, &vers) == 0) {
2720 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2721 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2722 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2726 sc->flags |= FW_UPTODATE;
2728 free(fw_data, M_DEVBUF);
2732 case CHELSIO_LOAD_BOOT: {
2734 struct ch_mem_range *t = (struct ch_mem_range *)data;
2736 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2740 error = copyin(t->buf, boot_data, t->len);
2742 error = -t3_load_boot(sc, boot_data, t->len);
2744 free(boot_data, M_DEVBUF);
2747 case CHELSIO_GET_PM: {
2748 struct ch_pm *m = (struct ch_pm *)data;
2749 struct tp_params *p = &sc->params.tp;
2751 if (!is_offload(sc))
2752 return (EOPNOTSUPP);
2754 m->tx_pg_sz = p->tx_pg_size;
2755 m->tx_num_pg = p->tx_num_pgs;
2756 m->rx_pg_sz = p->rx_pg_size;
2757 m->rx_num_pg = p->rx_num_pgs;
2758 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2762 case CHELSIO_SET_PM: {
2763 struct ch_pm *m = (struct ch_pm *)data;
2764 struct tp_params *p = &sc->params.tp;
2766 if (!is_offload(sc))
2767 return (EOPNOTSUPP);
2768 if (sc->flags & FULL_INIT_DONE)
2771 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2772 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2773 return (EINVAL); /* not power of 2 */
2774 if (!(m->rx_pg_sz & 0x14000))
2775 return (EINVAL); /* not 16KB or 64KB */
2776 if (!(m->tx_pg_sz & 0x1554000))
2778 if (m->tx_num_pg == -1)
2779 m->tx_num_pg = p->tx_num_pgs;
2780 if (m->rx_num_pg == -1)
2781 m->rx_num_pg = p->rx_num_pgs;
2782 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2784 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2785 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2788 p->rx_pg_size = m->rx_pg_sz;
2789 p->tx_pg_size = m->tx_pg_sz;
2790 p->rx_num_pgs = m->rx_num_pg;
2791 p->tx_num_pgs = m->tx_num_pg;
2794 case CHELSIO_SETMTUTAB: {
2795 struct ch_mtus *m = (struct ch_mtus *)data;
2798 if (!is_offload(sc))
2799 return (EOPNOTSUPP);
2800 if (offload_running(sc))
2802 if (m->nmtus != NMTUS)
2804 if (m->mtus[0] < 81) /* accommodate SACK */
2808 * MTUs must be in ascending order
2810 for (i = 1; i < NMTUS; ++i)
2811 if (m->mtus[i] < m->mtus[i - 1])
2814 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2817 case CHELSIO_GETMTUTAB: {
2818 struct ch_mtus *m = (struct ch_mtus *)data;
2820 if (!is_offload(sc))
2821 return (EOPNOTSUPP);
2823 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2827 case CHELSIO_GET_MEM: {
2828 struct ch_mem_range *t = (struct ch_mem_range *)data;
2834 * Use these to avoid modifying len/addr in the return
2837 uint32_t len = t->len, addr = t->addr;
2839 if (!is_offload(sc))
2840 return (EOPNOTSUPP);
2841 if (!(sc->flags & FULL_INIT_DONE))
2842 return (EIO); /* need the memory controllers */
2843 if ((addr & 0x7) || (len & 0x7))
2845 if (t->mem_id == MEM_CM)
2847 else if (t->mem_id == MEM_PMRX)
2849 else if (t->mem_id == MEM_PMTX)
2856 * bits 0..9: chip version
2857 * bits 10..15: chip revision
2859 t->version = 3 | (sc->params.rev << 10);
2862 * Read 256 bytes at a time as len can be large and we don't
2863 * want to use huge intermediate buffers.
2865 useraddr = (uint8_t *)t->buf;
2867 unsigned int chunk = min(len, sizeof(buf));
2869 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2872 if (copyout(buf, useraddr, chunk))
2880 case CHELSIO_READ_TCAM_WORD: {
2881 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2883 if (!is_offload(sc))
2884 return (EOPNOTSUPP);
2885 if (!(sc->flags & FULL_INIT_DONE))
2886 return (EIO); /* need MC5 */
2887 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2890 case CHELSIO_SET_TRACE_FILTER: {
2891 struct ch_trace *t = (struct ch_trace *)data;
2892 const struct trace_params *tp;
2894 tp = (const struct trace_params *)&t->sip;
2896 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2899 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2903 case CHELSIO_SET_PKTSCHED: {
2904 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2905 if (sc->open_device_map == 0)
2907 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2911 case CHELSIO_IFCONF_GETREGS: {
2912 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2913 int reglen = cxgb_get_regs_len();
2914 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2918 if (regs->len > reglen)
2920 else if (regs->len < reglen)
2924 cxgb_get_regs(sc, regs, buf);
2925 error = copyout(buf, regs->data, reglen);
2927 free(buf, M_DEVBUF);
2931 case CHELSIO_SET_HW_SCHED: {
2932 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2933 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2935 if ((sc->flags & FULL_INIT_DONE) == 0)
2936 return (EAGAIN); /* need TP to be initialized */
2937 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2938 !in_range(t->channel, 0, 1) ||
2939 !in_range(t->kbps, 0, 10000000) ||
2940 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2941 !in_range(t->flow_ipg, 0,
2942 dack_ticks_to_usec(sc, 0x7ff)))
2946 error = t3_config_sched(sc, t->kbps, t->sched);
2950 if (t->class_ipg >= 0)
2951 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2952 if (t->flow_ipg >= 0) {
2953 t->flow_ipg *= 1000; /* us -> ns */
2954 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2957 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2959 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2960 bit, t->mode ? bit : 0);
2962 if (t->channel >= 0)
2963 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2964 1 << t->sched, t->channel << t->sched);
2967 case CHELSIO_GET_EEPROM: {
2969 struct ch_eeprom *e = (struct ch_eeprom *)data;
2972 if (e->offset & 3 || e->offset >= EEPROMSIZE ||
2973 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
2977 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2981 e->magic = EEPROM_MAGIC;
2982 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2983 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2986 error = copyout(buf + e->offset, e->data, e->len);
2988 free(buf, M_DEVBUF);
2991 case CHELSIO_CLEAR_STATS: {
2992 if (!(sc->flags & FULL_INIT_DONE))
2996 t3_mac_update_stats(&pi->mac);
2997 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
3001 case CHELSIO_GET_UP_LA: {
3002 struct ch_up_la *la = (struct ch_up_la *)data;
3003 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
3007 if (la->bufsize < LA_BUFSIZE)
3011 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
3014 error = copyout(buf, la->data, la->bufsize);
3016 free(buf, M_DEVBUF);
3019 case CHELSIO_GET_UP_IOQS: {
3020 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3021 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3027 if (ioqs->bufsize < IOQS_BUFSIZE)
3031 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3034 v = (uint32_t *)buf;
3036 ioqs->ioq_rx_enable = *v++;
3037 ioqs->ioq_tx_enable = *v++;
3038 ioqs->ioq_rx_status = *v++;
3039 ioqs->ioq_tx_status = *v++;
3041 error = copyout(v, ioqs->data, ioqs->bufsize);
3044 free(buf, M_DEVBUF);
3047 case CHELSIO_SET_FILTER: {
3048 struct ch_filter *f = (struct ch_filter *)data;
3049 struct filter_info *p;
3050 unsigned int nfilters = sc->params.mc5.nfilters;
3052 if (!is_offload(sc))
3053 return (EOPNOTSUPP); /* No TCAM */
3054 if (!(sc->flags & FULL_INIT_DONE))
3055 return (EAGAIN); /* mc5 not setup yet */
3057 return (EBUSY); /* TOE will use TCAM */
3060 if (f->filter_id >= nfilters ||
3061 (f->val.dip && f->mask.dip != 0xffffffff) ||
3062 (f->val.sport && f->mask.sport != 0xffff) ||
3063 (f->val.dport && f->mask.dport != 0xffff) ||
3064 (f->val.vlan && f->mask.vlan != 0xfff) ||
3065 (f->val.vlan_prio &&
3066 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3067 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3068 f->qset >= SGE_QSETS ||
3069 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3072 /* Was allocated with M_WAITOK */
3073 KASSERT(sc->filters, ("filter table NULL\n"));
3075 p = &sc->filters[f->filter_id];
3079 bzero(p, sizeof(*p));
3080 p->sip = f->val.sip;
3081 p->sip_mask = f->mask.sip;
3082 p->dip = f->val.dip;
3083 p->sport = f->val.sport;
3084 p->dport = f->val.dport;
3085 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3086 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3088 p->mac_hit = f->mac_hit;
3089 p->mac_vld = f->mac_addr_idx != 0xffff;
3090 p->mac_idx = f->mac_addr_idx;
3091 p->pkt_type = f->proto;
3092 p->report_filter_id = f->want_filter_id;
3097 error = set_filter(sc, f->filter_id, p);
3102 case CHELSIO_DEL_FILTER: {
3103 struct ch_filter *f = (struct ch_filter *)data;
3104 struct filter_info *p;
3105 unsigned int nfilters = sc->params.mc5.nfilters;
3107 if (!is_offload(sc))
3108 return (EOPNOTSUPP);
3109 if (!(sc->flags & FULL_INIT_DONE))
3111 if (nfilters == 0 || sc->filters == NULL)
3113 if (f->filter_id >= nfilters)
3116 p = &sc->filters[f->filter_id];
3120 return (EFAULT); /* Read "Bad address" as "Bad index" */
3122 bzero(p, sizeof(*p));
3123 p->sip = p->sip_mask = 0xffffffff;
3125 p->vlan_prio = FILTER_NO_VLAN_PRI;
3127 error = set_filter(sc, f->filter_id, p);
3130 case CHELSIO_GET_FILTER: {
3131 struct ch_filter *f = (struct ch_filter *)data;
3132 struct filter_info *p;
3133 unsigned int i, nfilters = sc->params.mc5.nfilters;
3135 if (!is_offload(sc))
3136 return (EOPNOTSUPP);
3137 if (!(sc->flags & FULL_INIT_DONE))
3139 if (nfilters == 0 || sc->filters == NULL)
3142 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3143 for (; i < nfilters; i++) {
3144 p = &sc->filters[i];
3148 bzero(f, sizeof(*f));
3151 f->val.sip = p->sip;
3152 f->mask.sip = p->sip_mask;
3153 f->val.dip = p->dip;
3154 f->mask.dip = p->dip ? 0xffffffff : 0;
3155 f->val.sport = p->sport;
3156 f->mask.sport = p->sport ? 0xffff : 0;
3157 f->val.dport = p->dport;
3158 f->mask.dport = p->dport ? 0xffff : 0;
3159 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3160 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3161 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3163 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3164 0 : FILTER_NO_VLAN_PRI;
3165 f->mac_hit = p->mac_hit;
3166 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3167 f->proto = p->pkt_type;
3168 f->want_filter_id = p->report_filter_id;
3177 f->filter_id = 0xffffffff;
3181 return (EOPNOTSUPP);
3188 static __inline void
3189 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3192 uint32_t *p = (uint32_t *)(buf + start);
3194 for ( ; start <= end; start += sizeof(uint32_t))
3195 *p++ = t3_read_reg(ap, start);
3198 #define T3_REGMAP_SIZE (3 * 1024)
3200 cxgb_get_regs_len(void)
3202 return T3_REGMAP_SIZE;
3206 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3211 * bits 0..9: chip version
3212 * bits 10..15: chip revision
3213 * bit 31: set for PCIe cards
3215 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3218 * We skip the MAC statistics registers because they are clear-on-read.
3219 * Also reading multi-register stats would need to synchronize with the
3220 * periodic mac stats accumulation. Hard to justify the complexity.
3222 memset(buf, 0, cxgb_get_regs_len());
3223 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3224 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3225 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3226 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3227 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3228 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3229 XGM_REG(A_XGM_SERDES_STAT3, 1));
3230 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3231 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3235 alloc_filters(struct adapter *sc)
3237 struct filter_info *p;
3238 unsigned int nfilters = sc->params.mc5.nfilters;
3243 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3246 p = &sc->filters[nfilters - 1];
3248 p->vlan_prio = FILTER_NO_VLAN_PRI;
3249 p->pass = p->rss = p->valid = p->locked = 1;
3255 setup_hw_filters(struct adapter *sc)
3258 unsigned int nfilters = sc->params.mc5.nfilters;
3263 t3_enable_filters(sc);
3265 for (i = rc = 0; i < nfilters && !rc; i++) {
3266 if (sc->filters[i].locked)
3267 rc = set_filter(sc, i, &sc->filters[i]);
3274 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3278 struct ulp_txpkt *txpkt;
3279 struct work_request_hdr *wr;
3280 struct cpl_pass_open_req *oreq;
3281 struct cpl_set_tcb_field *sreq;
3283 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3284 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3286 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3287 sc->params.mc5.nfilters;
3289 m = m_gethdr(M_WAITOK, MT_DATA);
3290 m->m_len = m->m_pkthdr.len = len;
3291 bzero(mtod(m, char *), len);
3293 wr = mtod(m, struct work_request_hdr *);
3294 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3296 oreq = (struct cpl_pass_open_req *)(wr + 1);
3297 txpkt = (struct ulp_txpkt *)oreq;
3298 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3299 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3300 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3301 oreq->local_port = htons(f->dport);
3302 oreq->peer_port = htons(f->sport);
3303 oreq->local_ip = htonl(f->dip);
3304 oreq->peer_ip = htonl(f->sip);
3305 oreq->peer_netmask = htonl(f->sip_mask);
3307 oreq->opt0l = htonl(F_NO_OFFLOAD);
3308 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3309 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3310 V_VLAN_PRI(f->vlan_prio >> 1) |
3311 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3312 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3313 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3315 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3316 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3317 (f->report_filter_id << 15) | (1 << 23) |
3318 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3319 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3322 if (f->pass && !f->rss) {
3323 len = sizeof(*sreq);
3324 m = m_gethdr(M_WAITOK, MT_DATA);
3325 m->m_len = m->m_pkthdr.len = len;
3326 bzero(mtod(m, char *), len);
3327 sreq = mtod(m, struct cpl_set_tcb_field *);
3328 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3329 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3330 (u64)sc->rrss_map[f->qset] << 19);
3337 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3338 unsigned int word, u64 mask, u64 val)
3340 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3341 req->reply = V_NO_REPLY(1);
3343 req->word = htons(word);
3344 req->mask = htobe64(mask);
3345 req->val = htobe64(val);
3349 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3350 unsigned int word, u64 mask, u64 val)
3352 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3354 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3355 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3356 mk_set_tcb_field(req, tid, word, mask, val);
3360 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3364 mtx_lock(&t3_list_lock);
3365 SLIST_FOREACH(sc, &t3_list, link) {
3367 * func should not make any assumptions about what state sc is
3368 * in - the only guarantee is that sc->sc_lock is a valid lock.
3372 mtx_unlock(&t3_list_lock);
3377 toe_capability(struct port_info *pi, int enable)
3380 struct adapter *sc = pi->adapter;
3382 ADAPTER_LOCK_ASSERT_OWNED(sc);
3384 if (!is_offload(sc))
3388 if (!(sc->flags & FULL_INIT_DONE)) {
3390 "You must enable a cxgb interface first\n");
3394 if (isset(&sc->offload_map, pi->port_id))
3397 if (!(sc->flags & TOM_INIT_DONE)) {
3398 rc = t3_activate_uld(sc, ULD_TOM);
3401 "You must kldload t3_tom.ko before trying "
3402 "to enable TOE on a cxgb interface.\n");
3406 KASSERT(sc->tom_softc != NULL,
3407 ("%s: TOM activated but softc NULL", __func__));
3408 KASSERT(sc->flags & TOM_INIT_DONE,
3409 ("%s: TOM activated but flag not set", __func__));
3412 setbit(&sc->offload_map, pi->port_id);
3415 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3416 * enabled on any port. Need to figure out how to enable,
3417 * disable, load, and unload iWARP cleanly.
3419 if (!isset(&sc->offload_map, MAX_NPORTS) &&
3420 t3_activate_uld(sc, ULD_IWARP) == 0)
3421 setbit(&sc->offload_map, MAX_NPORTS);
3423 if (!isset(&sc->offload_map, pi->port_id))
3426 KASSERT(sc->flags & TOM_INIT_DONE,
3427 ("%s: TOM never initialized?", __func__));
3428 clrbit(&sc->offload_map, pi->port_id);
3435 * Add an upper layer driver to the global list.
3438 t3_register_uld(struct uld_info *ui)
3443 mtx_lock(&t3_uld_list_lock);
3444 SLIST_FOREACH(u, &t3_uld_list, link) {
3445 if (u->uld_id == ui->uld_id) {
3451 SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3454 mtx_unlock(&t3_uld_list_lock);
3459 t3_unregister_uld(struct uld_info *ui)
3464 mtx_lock(&t3_uld_list_lock);
3466 SLIST_FOREACH(u, &t3_uld_list, link) {
3468 if (ui->refcount > 0) {
3473 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3479 mtx_unlock(&t3_uld_list_lock);
3484 t3_activate_uld(struct adapter *sc, int id)
3487 struct uld_info *ui;
3489 mtx_lock(&t3_uld_list_lock);
3491 SLIST_FOREACH(ui, &t3_uld_list, link) {
3492 if (ui->uld_id == id) {
3493 rc = ui->activate(sc);
3500 mtx_unlock(&t3_uld_list_lock);
3506 t3_deactivate_uld(struct adapter *sc, int id)
3509 struct uld_info *ui;
3511 mtx_lock(&t3_uld_list_lock);
3513 SLIST_FOREACH(ui, &t3_uld_list, link) {
3514 if (ui->uld_id == id) {
3515 rc = ui->deactivate(sc);
3522 mtx_unlock(&t3_uld_list_lock);
3528 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3536 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3538 uintptr_t *loc, new;
3540 if (opcode >= NUM_CPL_HANDLERS)
3543 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3544 loc = (uintptr_t *) &sc->cpl_handler[opcode];
3545 atomic_store_rel_ptr(loc, new);
3552 cxgbc_mod_event(module_t mod, int cmd, void *arg)
3558 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3559 SLIST_INIT(&t3_list);
3561 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3562 SLIST_INIT(&t3_uld_list);
3568 mtx_lock(&t3_uld_list_lock);
3569 if (!SLIST_EMPTY(&t3_uld_list)) {
3571 mtx_unlock(&t3_uld_list_lock);
3574 mtx_unlock(&t3_uld_list_lock);
3575 mtx_destroy(&t3_uld_list_lock);
3577 mtx_lock(&t3_list_lock);
3578 if (!SLIST_EMPTY(&t3_list)) {
3580 mtx_unlock(&t3_list_lock);
3583 mtx_unlock(&t3_list_lock);
3584 mtx_destroy(&t3_list_lock);
3593 cxgb_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
3595 struct port_info *pi;
3598 pi = if_getsoftc(ifp);
3602 *ncl = adap->sge.qs[0].fl[1].size;
3603 *clsize = adap->sge.qs[0].fl[1].buf_size;
3604 ADAPTER_UNLOCK(adap);
3608 cxgb_netdump_event(struct ifnet *ifp, enum netdump_ev event)
3610 struct port_info *pi;
3611 struct sge_qset *qs;
3614 pi = if_getsoftc(ifp);
3615 if (event == NETDUMP_START)
3616 for (i = 0; i < SGE_QSETS; i++) {
3617 qs = &pi->adapter->sge.qs[i];
3619 /* Need to reinit after netdump_mbuf_dump(). */
3620 qs->fl[0].zone = zone_pack;
3621 qs->fl[1].zone = zone_clust;
3622 qs->lro.enabled = 0;
3627 cxgb_netdump_transmit(struct ifnet *ifp, struct mbuf *m)
3629 struct port_info *pi;
3630 struct sge_qset *qs;
3632 pi = if_getsoftc(ifp);
3633 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
3637 qs = &pi->adapter->sge.qs[pi->first_qset];
3638 return (cxgb_netdump_encap(qs, &m));
3642 cxgb_netdump_poll(struct ifnet *ifp, int count)
3644 struct port_info *pi;
3648 pi = if_getsoftc(ifp);
3649 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
3653 for (i = 0; i < SGE_QSETS; i++)
3654 (void)cxgb_netdump_poll_rx(adap, &adap->sge.qs[i]);
3655 (void)cxgb_netdump_poll_tx(&adap->sge.qs[pi->first_qset]);
3658 #endif /* NETDUMP */