1 /**************************************************************************
2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 Copyright (c) 2007-2009, Chelsio Inc.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
29 ***************************************************************************/
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
40 #include <sys/module.h>
41 #include <sys/pciio.h>
43 #include <machine/bus.h>
44 #include <machine/resource.h>
47 #include <sys/ioccom.h>
49 #include <sys/linker.h>
50 #include <sys/firmware.h>
51 #include <sys/socket.h>
52 #include <sys/sockio.h>
54 #include <sys/sysctl.h>
55 #include <sys/syslog.h>
56 #include <sys/queue.h>
57 #include <sys/taskqueue.h>
61 #include <net/ethernet.h>
63 #include <net/if_var.h>
64 #include <net/if_arp.h>
65 #include <net/if_dl.h>
66 #include <net/if_media.h>
67 #include <net/if_types.h>
68 #include <net/if_vlan_var.h>
70 #include <netinet/in_systm.h>
71 #include <netinet/in.h>
72 #include <netinet/if_ether.h>
73 #include <netinet/ip.h>
74 #include <netinet/ip.h>
75 #include <netinet/tcp.h>
76 #include <netinet/udp.h>
77 #include <netinet/netdump/netdump.h>
79 #include <dev/pci/pcireg.h>
80 #include <dev/pci/pcivar.h>
81 #include <dev/pci/pci_private.h>
83 #include <cxgb_include.h>
89 static int cxgb_setup_interrupts(adapter_t *);
90 static void cxgb_teardown_interrupts(adapter_t *);
91 static void cxgb_init(void *);
92 static int cxgb_init_locked(struct port_info *);
93 static int cxgb_uninit_locked(struct port_info *);
94 static int cxgb_uninit_synchronized(struct port_info *);
95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
96 static int cxgb_media_change(struct ifnet *);
97 static int cxgb_ifm_type(int);
98 static void cxgb_build_medialist(struct port_info *);
99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
100 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
101 static int setup_sge_qsets(adapter_t *);
102 static void cxgb_async_intr(void *);
103 static void cxgb_tick_handler(void *, int);
104 static void cxgb_tick(void *);
105 static void link_check_callout(void *);
106 static void check_link_status(void *, int);
107 static void setup_rss(adapter_t *sc);
108 static int alloc_filters(struct adapter *);
109 static int setup_hw_filters(struct adapter *);
110 static int set_filter(struct adapter *, int, const struct filter_info *);
111 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
112 unsigned int, u64, u64);
113 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
114 unsigned int, u64, u64);
116 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
119 /* Attachment glue for the PCI controller end of the device. Each port of
120 * the device is attached separately, as defined later.
122 static int cxgb_controller_probe(device_t);
123 static int cxgb_controller_attach(device_t);
124 static int cxgb_controller_detach(device_t);
125 static void cxgb_free(struct adapter *);
126 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
128 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
129 static int cxgb_get_regs_len(void);
130 static void touch_bars(device_t dev);
131 static void cxgb_update_mac_settings(struct port_info *p);
133 static int toe_capability(struct port_info *, int);
136 /* Table for probing the cards. The desc field isn't actually used */
142 } cxgb_identifiers[] = {
143 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
144 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
145 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
146 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
147 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
148 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
149 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
150 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
151 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
152 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
153 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
154 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
155 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
156 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
160 static device_method_t cxgb_controller_methods[] = {
161 DEVMETHOD(device_probe, cxgb_controller_probe),
162 DEVMETHOD(device_attach, cxgb_controller_attach),
163 DEVMETHOD(device_detach, cxgb_controller_detach),
168 static driver_t cxgb_controller_driver = {
170 cxgb_controller_methods,
171 sizeof(struct adapter)
174 static int cxgbc_mod_event(module_t, int, void *);
175 static devclass_t cxgb_controller_devclass;
176 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
178 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
179 nitems(cxgb_identifiers) - 1);
180 MODULE_VERSION(cxgbc, 1);
181 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
184 * Attachment glue for the ports. Attachment is done directly to the
187 static int cxgb_port_probe(device_t);
188 static int cxgb_port_attach(device_t);
189 static int cxgb_port_detach(device_t);
191 static device_method_t cxgb_port_methods[] = {
192 DEVMETHOD(device_probe, cxgb_port_probe),
193 DEVMETHOD(device_attach, cxgb_port_attach),
194 DEVMETHOD(device_detach, cxgb_port_detach),
198 static driver_t cxgb_port_driver = {
204 static d_ioctl_t cxgb_extension_ioctl;
205 static d_open_t cxgb_extension_open;
206 static d_close_t cxgb_extension_close;
208 static struct cdevsw cxgb_cdevsw = {
209 .d_version = D_VERSION,
211 .d_open = cxgb_extension_open,
212 .d_close = cxgb_extension_close,
213 .d_ioctl = cxgb_extension_ioctl,
217 static devclass_t cxgb_port_devclass;
218 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
219 MODULE_VERSION(cxgb, 1);
221 NETDUMP_DEFINE(cxgb);
223 static struct mtx t3_list_lock;
224 static SLIST_HEAD(, adapter) t3_list;
226 static struct mtx t3_uld_list_lock;
227 static SLIST_HEAD(, uld_info) t3_uld_list;
231 * The driver uses the best interrupt scheme available on a platform in the
232 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
233 * of these schemes the driver may consider as follows:
235 * msi = 2: choose from among all three options
236 * msi = 1 : only consider MSI and pin interrupts
237 * msi = 0: force pin interrupts
239 static int msi_allowed = 2;
241 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
242 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
243 "MSI-X, MSI, INTx selector");
246 * The driver uses an auto-queue algorithm by default.
247 * To disable it and force a single queue-set per port, use multiq = 0
249 static int multiq = 1;
250 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
251 "use min(ncpus/ports, 8) queue-sets per port");
254 * By default the driver will not update the firmware unless
255 * it was compiled against a newer version
258 static int force_fw_update = 0;
259 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
260 "update firmware even if up to date");
262 int cxgb_use_16k_clusters = -1;
263 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
264 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
266 static int nfilters = -1;
267 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
268 &nfilters, 0, "max number of entries in the filter table");
271 MAX_TXQ_ENTRIES = 16384,
272 MAX_CTRL_TXQ_ENTRIES = 1024,
273 MAX_RSPQ_ENTRIES = 16384,
274 MAX_RX_BUFFERS = 16384,
275 MAX_RX_JUMBO_BUFFERS = 16384,
277 MIN_CTRL_TXQ_ENTRIES = 4,
278 MIN_RSPQ_ENTRIES = 32,
280 MIN_FL_JUMBO_ENTRIES = 32
295 u32 report_filter_id:1;
303 enum { FILTER_NO_VLAN_PRI = 7 };
305 #define EEPROM_MAGIC 0x38E2F10C
307 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
310 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
314 t3rev2char(struct adapter *adapter)
318 switch(adapter->params.rev) {
333 static struct cxgb_ident *
334 cxgb_get_ident(device_t dev)
336 struct cxgb_ident *id;
338 for (id = cxgb_identifiers; id->desc != NULL; id++) {
339 if ((id->vendor == pci_get_vendor(dev)) &&
340 (id->device == pci_get_device(dev))) {
347 static const struct adapter_info *
348 cxgb_get_adapter_info(device_t dev)
350 struct cxgb_ident *id;
351 const struct adapter_info *ai;
353 id = cxgb_get_ident(dev);
357 ai = t3_get_adapter_info(id->index);
363 cxgb_controller_probe(device_t dev)
365 const struct adapter_info *ai;
366 char *ports, buf[80];
369 ai = cxgb_get_adapter_info(dev);
373 nports = ai->nports0 + ai->nports1;
379 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
380 device_set_desc_copy(dev, buf);
381 return (BUS_PROBE_DEFAULT);
384 #define FW_FNAME "cxgb_t3fw"
385 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
386 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
389 upgrade_fw(adapter_t *sc)
391 const struct firmware *fw;
395 if ((fw = firmware_get(FW_FNAME)) == NULL) {
396 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
399 device_printf(sc->dev, "installing firmware on card\n");
400 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
403 device_printf(sc->dev, "failed to install firmware: %d\n",
406 t3_get_fw_version(sc, &vers);
407 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
408 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
409 G_FW_VERSION_MICRO(vers));
412 firmware_put(fw, FIRMWARE_UNLOAD);
418 * The cxgb_controller_attach function is responsible for the initial
419 * bringup of the device. Its responsibilities include:
421 * 1. Determine if the device supports MSI or MSI-X.
422 * 2. Allocate bus resources so that we can access the Base Address Register
423 * 3. Create and initialize mutexes for the controller and its control
424 * logic such as SGE and MDIO.
425 * 4. Call hardware specific setup routine for the adapter as a whole.
426 * 5. Allocate the BAR for doing MSI-X.
427 * 6. Setup the line interrupt iff MSI-X is not supported.
428 * 7. Create the driver's taskq.
429 * 8. Start one task queue service thread.
430 * 9. Check if the firmware and SRAM are up-to-date. They will be
431 * auto-updated later (before FULL_INIT_DONE), if required.
432 * 10. Create a child device for each MAC (port)
433 * 11. Initialize T3 private state.
434 * 12. Trigger the LED
435 * 13. Setup offload iff supported.
436 * 14. Reset/restart the tick callout.
439 * NOTE: Any modification or deviation from this list MUST be reflected in
440 * the above comment. Failure to do so will result in problems on various
441 * error conditions including link flapping.
444 cxgb_controller_attach(device_t dev)
447 const struct adapter_info *ai;
455 sc = device_get_softc(dev);
458 ai = cxgb_get_adapter_info(dev);
460 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
461 device_get_unit(dev));
462 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
464 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
465 device_get_unit(dev));
466 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
467 device_get_unit(dev));
468 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
469 device_get_unit(dev));
471 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
472 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
473 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
475 mtx_lock(&t3_list_lock);
476 SLIST_INSERT_HEAD(&t3_list, sc, link);
477 mtx_unlock(&t3_list_lock);
479 /* find the PCIe link width and set max read request to 4KB*/
480 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) {
483 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
484 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
485 if (sc->link_width < 8 &&
486 (ai->caps & SUPPORTED_10000baseT_Full)) {
487 device_printf(sc->dev,
488 "PCIe x%d Link, expect reduced performance\n",
492 pci_set_max_read_req(dev, 4096);
496 pci_enable_busmaster(dev);
498 * Allocate the registers and make them available to the driver.
499 * The registers that we care about for NIC mode are in BAR 0
501 sc->regs_rid = PCIR_BAR(0);
502 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
503 &sc->regs_rid, RF_ACTIVE)) == NULL) {
504 device_printf(dev, "Cannot allocate BAR region 0\n");
509 sc->bt = rman_get_bustag(sc->regs_res);
510 sc->bh = rman_get_bushandle(sc->regs_res);
511 sc->mmio_len = rman_get_size(sc->regs_res);
513 for (i = 0; i < MAX_NPORTS; i++)
514 sc->port[i].adapter = sc;
516 if (t3_prep_adapter(sc, ai, 1) < 0) {
517 printf("prep adapter failed\n");
522 sc->udbs_rid = PCIR_BAR(2);
524 if (is_offload(sc) &&
525 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
526 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
527 device_printf(dev, "Cannot allocate BAR region 1\n");
532 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
533 * enough messages for the queue sets. If that fails, try falling
534 * back to MSI. If that fails, then try falling back to the legacy
535 * interrupt pin model.
537 sc->msix_regs_rid = 0x20;
538 if ((msi_allowed >= 2) &&
539 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
540 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
543 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
544 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
546 if (pci_msix_count(dev) == 0 ||
547 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
548 sc->msi_count != msi_needed) {
549 device_printf(dev, "alloc msix failed - "
550 "msi_count=%d, msi_needed=%d, err=%d; "
551 "will try MSI\n", sc->msi_count,
555 pci_release_msi(dev);
556 bus_release_resource(dev, SYS_RES_MEMORY,
557 sc->msix_regs_rid, sc->msix_regs_res);
558 sc->msix_regs_res = NULL;
560 sc->flags |= USING_MSIX;
561 sc->cxgb_intr = cxgb_async_intr;
563 "using MSI-X interrupts (%u vectors)\n",
568 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
570 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
571 device_printf(dev, "alloc msi failed - "
572 "err=%d; will try INTx\n", error);
575 pci_release_msi(dev);
577 sc->flags |= USING_MSI;
578 sc->cxgb_intr = t3_intr_msi;
579 device_printf(dev, "using MSI interrupts\n");
582 if (sc->msi_count == 0) {
583 device_printf(dev, "using line interrupts\n");
584 sc->cxgb_intr = t3b_intr;
587 /* Create a private taskqueue thread for handling driver events */
588 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
589 taskqueue_thread_enqueue, &sc->tq);
590 if (sc->tq == NULL) {
591 device_printf(dev, "failed to allocate controller task queue\n");
595 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
596 device_get_nameunit(dev));
597 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
600 /* Create a periodic callout for checking adapter status */
601 callout_init(&sc->cxgb_tick_ch, 1);
603 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
605 * Warn user that a firmware update will be attempted in init.
607 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
608 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
609 sc->flags &= ~FW_UPTODATE;
611 sc->flags |= FW_UPTODATE;
614 if (t3_check_tpsram_version(sc) < 0) {
616 * Warn user that a firmware update will be attempted in init.
618 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
619 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
620 sc->flags &= ~TPS_UPTODATE;
622 sc->flags |= TPS_UPTODATE;
626 * Create a child device for each MAC. The ethernet attachment
627 * will be done in these children.
629 for (i = 0; i < (sc)->params.nports; i++) {
630 struct port_info *pi;
632 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
633 device_printf(dev, "failed to add child port\n");
639 pi->nqsets = port_qsets;
640 pi->first_qset = i*port_qsets;
642 pi->tx_chan = i >= ai->nports0;
643 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
644 sc->rxpkt_map[pi->txpkt_intf] = i;
645 sc->port[i].tx_chan = i >= ai->nports0;
646 sc->portdev[i] = child;
647 device_set_softc(child, pi);
649 if ((error = bus_generic_attach(dev)) != 0)
652 /* initialize sge private state */
653 t3_sge_init_adapter(sc);
657 error = t3_get_fw_version(sc, &vers);
661 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
662 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
663 G_FW_VERSION_MICRO(vers));
665 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
666 ai->desc, is_offload(sc) ? "R" : "",
667 sc->params.vpd.ec, sc->params.vpd.sn);
668 device_set_desc_copy(dev, buf);
670 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
671 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
672 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
674 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
675 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
676 t3_add_attach_sysctls(sc);
679 for (i = 0; i < NUM_CPL_HANDLERS; i++)
680 sc->cpl_handler[i] = cpl_not_handled;
684 error = cxgb_setup_interrupts(sc);
693 * The cxgb_controller_detach routine is called with the device is
694 * unloaded from the system.
698 cxgb_controller_detach(device_t dev)
702 sc = device_get_softc(dev);
710 * The cxgb_free() is called by the cxgb_controller_detach() routine
711 * to tear down the structures that were built up in
712 * cxgb_controller_attach(), and should be the final piece of work
713 * done when fully unloading the driver.
716 * 1. Shutting down the threads started by the cxgb_controller_attach()
718 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
719 * 3. Detaching all of the port devices created during the
720 * cxgb_controller_attach() routine.
721 * 4. Removing the device children created via cxgb_controller_attach().
722 * 5. Releasing PCI resources associated with the device.
723 * 6. Turning off the offload support, iff it was turned on.
724 * 7. Destroying the mutexes created in cxgb_controller_attach().
728 cxgb_free(struct adapter *sc)
733 sc->flags |= CXGB_SHUTDOWN;
737 * Make sure all child devices are gone.
739 bus_generic_detach(sc->dev);
740 for (i = 0; i < (sc)->params.nports; i++) {
741 if (sc->portdev[i] &&
742 device_delete_child(sc->dev, sc->portdev[i]) != 0)
743 device_printf(sc->dev, "failed to delete child port\n");
744 nqsets += sc->port[i].nqsets;
748 * At this point, it is as if cxgb_port_detach has run on all ports, and
749 * cxgb_down has run on the adapter. All interrupts have been silenced,
750 * all open devices have been closed.
752 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
753 __func__, sc->open_device_map));
754 for (i = 0; i < sc->params.nports; i++) {
755 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
760 * Finish off the adapter's callouts.
762 callout_drain(&sc->cxgb_tick_ch);
763 callout_drain(&sc->sge_timer_ch);
766 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
767 * sysctls are cleaned up by the kernel linker.
769 if (sc->flags & FULL_INIT_DONE) {
770 t3_free_sge_resources(sc, nqsets);
771 sc->flags &= ~FULL_INIT_DONE;
775 * Release all interrupt resources.
777 cxgb_teardown_interrupts(sc);
778 if (sc->flags & (USING_MSI | USING_MSIX)) {
779 device_printf(sc->dev, "releasing msi message(s)\n");
780 pci_release_msi(sc->dev);
782 device_printf(sc->dev, "no msi message to release\n");
785 if (sc->msix_regs_res != NULL) {
786 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
791 * Free the adapter's taskqueue.
793 if (sc->tq != NULL) {
794 taskqueue_free(sc->tq);
798 free(sc->filters, M_DEVBUF);
801 if (sc->udbs_res != NULL)
802 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
805 if (sc->regs_res != NULL)
806 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
809 MTX_DESTROY(&sc->mdio_lock);
810 MTX_DESTROY(&sc->sge.reg_lock);
811 MTX_DESTROY(&sc->elmer_lock);
812 mtx_lock(&t3_list_lock);
813 SLIST_REMOVE(&t3_list, sc, adapter, link);
814 mtx_unlock(&t3_list_lock);
815 ADAPTER_LOCK_DEINIT(sc);
819 * setup_sge_qsets - configure SGE Tx/Rx/response queues
820 * @sc: the controller softc
822 * Determines how many sets of SGE queues to use and initializes them.
823 * We support multiple queue sets per port if we have MSI-X, otherwise
824 * just one queue set per port.
827 setup_sge_qsets(adapter_t *sc)
829 int i, j, err, irq_idx = 0, qset_idx = 0;
830 u_int ntxq = SGE_TXQ_PER_SET;
832 if ((err = t3_sge_alloc(sc)) != 0) {
833 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
837 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
840 for (i = 0; i < (sc)->params.nports; i++) {
841 struct port_info *pi = &sc->port[i];
843 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
844 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
845 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
846 &sc->params.sge.qset[qset_idx], ntxq, pi);
848 t3_free_sge_resources(sc, qset_idx);
849 device_printf(sc->dev,
850 "t3_sge_alloc_qset failed with %d\n", err);
856 sc->nqsets = qset_idx;
862 cxgb_teardown_interrupts(adapter_t *sc)
866 for (i = 0; i < SGE_QSETS; i++) {
867 if (sc->msix_intr_tag[i] == NULL) {
869 /* Should have been setup fully or not at all */
870 KASSERT(sc->msix_irq_res[i] == NULL &&
871 sc->msix_irq_rid[i] == 0,
872 ("%s: half-done interrupt (%d).", __func__, i));
877 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
878 sc->msix_intr_tag[i]);
879 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
880 sc->msix_irq_res[i]);
882 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
883 sc->msix_irq_rid[i] = 0;
887 KASSERT(sc->irq_res != NULL,
888 ("%s: half-done interrupt.", __func__));
890 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
891 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
894 sc->irq_res = sc->intr_tag = NULL;
900 cxgb_setup_interrupts(adapter_t *sc)
902 struct resource *res;
904 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
906 sc->irq_rid = intr_flag ? 1 : 0;
907 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
908 RF_SHAREABLE | RF_ACTIVE);
909 if (sc->irq_res == NULL) {
910 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
911 intr_flag, sc->irq_rid);
915 err = bus_setup_intr(sc->dev, sc->irq_res,
916 INTR_MPSAFE | INTR_TYPE_NET, NULL,
917 sc->cxgb_intr, sc, &sc->intr_tag);
920 device_printf(sc->dev,
921 "Cannot set up interrupt (%x, %u, %d)\n",
922 intr_flag, sc->irq_rid, err);
923 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
925 sc->irq_res = sc->intr_tag = NULL;
930 /* That's all for INTx or MSI */
931 if (!(intr_flag & USING_MSIX) || err)
934 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
935 for (i = 0; i < sc->msi_count - 1; i++) {
937 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
938 RF_SHAREABLE | RF_ACTIVE);
940 device_printf(sc->dev, "Cannot allocate interrupt "
941 "for message %d\n", rid);
946 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
947 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
949 device_printf(sc->dev, "Cannot set up interrupt "
950 "for message %d (%d)\n", rid, err);
951 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
955 sc->msix_irq_rid[i] = rid;
956 sc->msix_irq_res[i] = res;
957 sc->msix_intr_tag[i] = tag;
958 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
962 cxgb_teardown_interrupts(sc);
969 cxgb_port_probe(device_t dev)
975 p = device_get_softc(dev);
977 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
978 device_set_desc_copy(dev, buf);
984 cxgb_makedev(struct port_info *pi)
987 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
988 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
990 if (pi->port_cdev == NULL)
993 pi->port_cdev->si_drv1 = (void *)pi;
998 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
999 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1000 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
1001 #define CXGB_CAP_ENABLE CXGB_CAP
1004 cxgb_port_attach(device_t dev)
1006 struct port_info *p;
1011 p = device_get_softc(dev);
1013 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1014 device_get_unit(device_get_parent(dev)), p->port_id);
1015 PORT_LOCK_INIT(p, p->lockbuf);
1017 callout_init(&p->link_check_ch, 1);
1018 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1020 /* Allocate an ifnet object and set it up */
1021 ifp = p->ifp = if_alloc(IFT_ETHER);
1023 device_printf(dev, "Cannot allocate ifnet\n");
1027 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1028 ifp->if_init = cxgb_init;
1030 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1031 ifp->if_ioctl = cxgb_ioctl;
1032 ifp->if_transmit = cxgb_transmit;
1033 ifp->if_qflush = cxgb_qflush;
1034 ifp->if_get_counter = cxgb_get_counter;
1036 ifp->if_capabilities = CXGB_CAP;
1039 ifp->if_capabilities |= IFCAP_TOE4;
1041 ifp->if_capenable = CXGB_CAP_ENABLE;
1042 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1043 CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1046 * Disable TSO on 4-port - it isn't supported by the firmware.
1048 if (sc->params.nports > 2) {
1049 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1050 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1051 ifp->if_hwassist &= ~CSUM_TSO;
1054 ether_ifattach(ifp, p->hw_addr);
1056 /* Attach driver netdump methods. */
1057 NETDUMP_SET(ifp, cxgb);
1059 #ifdef DEFAULT_JUMBO
1060 if (sc->params.nports <= 2)
1061 ifp->if_mtu = ETHERMTU_JUMBO;
1063 if ((err = cxgb_makedev(p)) != 0) {
1064 printf("makedev failed %d\n", err);
1068 /* Create a list of media supported by this port */
1069 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1071 cxgb_build_medialist(p);
1073 t3_sge_init_port(p);
1079 * cxgb_port_detach() is called via the device_detach methods when
1080 * cxgb_free() calls the bus_generic_detach. It is responsible for
1081 * removing the device from the view of the kernel, i.e. from all
1082 * interfaces lists etc. This routine is only called when the driver is
1083 * being unloaded, not when the link goes down.
1086 cxgb_port_detach(device_t dev)
1088 struct port_info *p;
1092 p = device_get_softc(dev);
1095 /* Tell cxgb_ioctl and if_init that the port is going away */
1100 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1104 if (p->port_cdev != NULL)
1105 destroy_dev(p->port_cdev);
1107 cxgb_uninit_synchronized(p);
1108 ether_ifdetach(p->ifp);
1110 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1111 struct sge_qset *qs = &sc->sge.qs[i];
1112 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1114 callout_drain(&txq->txq_watchdog);
1115 callout_drain(&txq->txq_timer);
1118 PORT_LOCK_DEINIT(p);
1124 wakeup_one(&sc->flags);
1130 t3_fatal_err(struct adapter *sc)
1134 if (sc->flags & FULL_INIT_DONE) {
1136 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1137 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1138 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1139 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1140 t3_intr_disable(sc);
1142 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1143 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1144 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1145 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1149 t3_os_find_pci_capability(adapter_t *sc, int cap)
1152 struct pci_devinfo *dinfo;
1158 dinfo = device_get_ivars(dev);
1161 status = pci_read_config(dev, PCIR_STATUS, 2);
1162 if (!(status & PCIM_STATUS_CAPPRESENT))
1165 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1171 ptr = PCIR_CAP_PTR_2;
1177 ptr = pci_read_config(dev, ptr, 1);
1180 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1182 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1189 t3_os_pci_save_state(struct adapter *sc)
1192 struct pci_devinfo *dinfo;
1195 dinfo = device_get_ivars(dev);
1197 pci_cfg_save(dev, dinfo, 0);
1202 t3_os_pci_restore_state(struct adapter *sc)
1205 struct pci_devinfo *dinfo;
1208 dinfo = device_get_ivars(dev);
1210 pci_cfg_restore(dev, dinfo);
1215 * t3_os_link_changed - handle link status changes
1216 * @sc: the adapter associated with the link change
1217 * @port_id: the port index whose link status has changed
1218 * @link_status: the new status of the link
1219 * @speed: the new speed setting
1220 * @duplex: the new duplex setting
1221 * @fc: the new flow-control setting
1223 * This is the OS-dependent handler for link status changes. The OS
1224 * neutral handler takes care of most of the processing for these events,
1225 * then calls this handler for any OS-specific processing.
1228 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1229 int duplex, int fc, int mac_was_reset)
1231 struct port_info *pi = &adapter->port[port_id];
1232 struct ifnet *ifp = pi->ifp;
1234 /* no race with detach, so ifp should always be good */
1235 KASSERT(ifp, ("%s: if detached.", __func__));
1237 /* Reapply mac settings if they were lost due to a reset */
1238 if (mac_was_reset) {
1240 cxgb_update_mac_settings(pi);
1245 ifp->if_baudrate = IF_Mbps(speed);
1246 if_link_state_change(ifp, LINK_STATE_UP);
1248 if_link_state_change(ifp, LINK_STATE_DOWN);
1252 * t3_os_phymod_changed - handle PHY module changes
1253 * @phy: the PHY reporting the module change
1254 * @mod_type: new module type
1256 * This is the OS-dependent handler for PHY module changes. It is
1257 * invoked when a PHY module is removed or inserted for any OS-specific
1260 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1262 static const char *mod_str[] = {
1263 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1265 struct port_info *pi = &adap->port[port_id];
1266 int mod = pi->phy.modtype;
1268 if (mod != pi->media.ifm_cur->ifm_data)
1269 cxgb_build_medialist(pi);
1271 if (mod == phy_modtype_none)
1272 if_printf(pi->ifp, "PHY module unplugged\n");
1274 KASSERT(mod < ARRAY_SIZE(mod_str),
1275 ("invalid PHY module type %d", mod));
1276 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1281 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1285 * The ifnet might not be allocated before this gets called,
1286 * as this is called early on in attach by t3_prep_adapter
1287 * save the address off in the port structure
1290 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1291 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1295 * Programs the XGMAC based on the settings in the ifnet. These settings
1296 * include MTU, MAC address, mcast addresses, etc.
1299 cxgb_update_mac_settings(struct port_info *p)
1301 struct ifnet *ifp = p->ifp;
1302 struct t3_rx_mode rm;
1303 struct cmac *mac = &p->mac;
1306 PORT_LOCK_ASSERT_OWNED(p);
1308 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1311 if (ifp->if_capenable & IFCAP_VLAN_MTU)
1312 mtu += ETHER_VLAN_ENCAP_LEN;
1314 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1316 t3_mac_set_mtu(mac, mtu);
1317 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1318 t3_mac_set_address(mac, 0, p->hw_addr);
1319 t3_init_rx_mode(&rm, p);
1320 t3_mac_set_rx_mode(mac, &rm);
1325 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1330 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1339 init_tp_parity(struct adapter *adap)
1343 struct cpl_set_tcb_field *greq;
1344 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1346 t3_tp_set_offload_mode(adap, 1);
1348 for (i = 0; i < 16; i++) {
1349 struct cpl_smt_write_req *req;
1351 m = m_gethdr(M_WAITOK, MT_DATA);
1352 req = mtod(m, struct cpl_smt_write_req *);
1353 m->m_len = m->m_pkthdr.len = sizeof(*req);
1354 memset(req, 0, sizeof(*req));
1355 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1356 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1358 t3_mgmt_tx(adap, m);
1361 for (i = 0; i < 2048; i++) {
1362 struct cpl_l2t_write_req *req;
1364 m = m_gethdr(M_WAITOK, MT_DATA);
1365 req = mtod(m, struct cpl_l2t_write_req *);
1366 m->m_len = m->m_pkthdr.len = sizeof(*req);
1367 memset(req, 0, sizeof(*req));
1368 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1369 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1370 req->params = htonl(V_L2T_W_IDX(i));
1371 t3_mgmt_tx(adap, m);
1374 for (i = 0; i < 2048; i++) {
1375 struct cpl_rte_write_req *req;
1377 m = m_gethdr(M_WAITOK, MT_DATA);
1378 req = mtod(m, struct cpl_rte_write_req *);
1379 m->m_len = m->m_pkthdr.len = sizeof(*req);
1380 memset(req, 0, sizeof(*req));
1381 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1382 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1383 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1384 t3_mgmt_tx(adap, m);
1387 m = m_gethdr(M_WAITOK, MT_DATA);
1388 greq = mtod(m, struct cpl_set_tcb_field *);
1389 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1390 memset(greq, 0, sizeof(*greq));
1391 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1392 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1393 greq->mask = htobe64(1);
1394 t3_mgmt_tx(adap, m);
1396 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1397 t3_tp_set_offload_mode(adap, 0);
1402 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1403 * @adap: the adapter
1405 * Sets up RSS to distribute packets to multiple receive queues. We
1406 * configure the RSS CPU lookup table to distribute to the number of HW
1407 * receive queues, and the response queue lookup table to narrow that
1408 * down to the response queues actually configured for each port.
1409 * We always configure the RSS mapping for two ports since the mapping
1410 * table has plenty of entries.
1413 setup_rss(adapter_t *adap)
1417 uint8_t cpus[SGE_QSETS + 1];
1418 uint16_t rspq_map[RSS_TABLE_SIZE];
1420 for (i = 0; i < SGE_QSETS; ++i)
1422 cpus[SGE_QSETS] = 0xff;
1425 for_each_port(adap, i) {
1426 const struct port_info *pi = adap2pinfo(adap, i);
1428 nq[pi->tx_chan] += pi->nqsets;
1430 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1431 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1432 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1435 /* Calculate the reverse RSS map table */
1436 for (i = 0; i < SGE_QSETS; ++i)
1437 adap->rrss_map[i] = 0xff;
1438 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1439 if (adap->rrss_map[rspq_map[i]] == 0xff)
1440 adap->rrss_map[rspq_map[i]] = i;
1442 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1443 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1444 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1449 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1453 struct mngt_pktsched_wr *req;
1455 m = m_gethdr(M_NOWAIT, MT_DATA);
1457 req = mtod(m, struct mngt_pktsched_wr *);
1458 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1459 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1464 req->binding = port;
1465 m->m_len = m->m_pkthdr.len = sizeof(*req);
1466 t3_mgmt_tx(adap, m);
1471 bind_qsets(adapter_t *sc)
1475 for (i = 0; i < (sc)->params.nports; ++i) {
1476 const struct port_info *pi = adap2pinfo(sc, i);
1478 for (j = 0; j < pi->nqsets; ++j) {
1479 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1487 update_tpeeprom(struct adapter *adap)
1489 const struct firmware *tpeeprom;
1492 unsigned int major, minor;
1496 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1498 major = G_TP_VERSION_MAJOR(version);
1499 minor = G_TP_VERSION_MINOR(version);
1500 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1503 rev = t3rev2char(adap);
1504 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1506 tpeeprom = firmware_get(name);
1507 if (tpeeprom == NULL) {
1508 device_printf(adap->dev,
1509 "could not load TP EEPROM: unable to load %s\n",
1514 len = tpeeprom->datasize - 4;
1516 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1518 goto release_tpeeprom;
1520 if (len != TP_SRAM_LEN) {
1521 device_printf(adap->dev,
1522 "%s length is wrong len=%d expected=%d\n", name,
1527 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1531 device_printf(adap->dev,
1532 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1533 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1535 device_printf(adap->dev,
1536 "Protocol SRAM image update in EEPROM failed\n");
1539 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1545 update_tpsram(struct adapter *adap)
1547 const struct firmware *tpsram;
1551 rev = t3rev2char(adap);
1552 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1554 update_tpeeprom(adap);
1556 tpsram = firmware_get(name);
1557 if (tpsram == NULL){
1558 device_printf(adap->dev, "could not load TP SRAM\n");
1561 device_printf(adap->dev, "updating TP SRAM\n");
1563 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1565 goto release_tpsram;
1567 ret = t3_set_proto_sram(adap, tpsram->data);
1569 device_printf(adap->dev, "loading protocol SRAM failed\n");
1572 firmware_put(tpsram, FIRMWARE_UNLOAD);
1578 * cxgb_up - enable the adapter
1579 * @adap: adapter being enabled
1581 * Called when the first port is enabled, this function performs the
1582 * actions necessary to make an adapter operational, such as completing
1583 * the initialization of HW modules, and enabling interrupts.
1586 cxgb_up(struct adapter *sc)
1589 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1591 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1592 __func__, sc->open_device_map));
1594 if ((sc->flags & FULL_INIT_DONE) == 0) {
1596 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1598 if ((sc->flags & FW_UPTODATE) == 0)
1599 if ((err = upgrade_fw(sc)))
1602 if ((sc->flags & TPS_UPTODATE) == 0)
1603 if ((err = update_tpsram(sc)))
1606 if (is_offload(sc) && nfilters != 0) {
1607 sc->params.mc5.nservers = 0;
1610 sc->params.mc5.nfilters = mxf;
1612 sc->params.mc5.nfilters = min(nfilters, mxf);
1615 err = t3_init_hw(sc, 0);
1619 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1620 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1622 err = setup_sge_qsets(sc);
1629 t3_add_configured_sysctls(sc);
1630 sc->flags |= FULL_INIT_DONE;
1637 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1638 is_offload(sc) && init_tp_parity(sc) == 0)
1639 sc->flags |= TP_PARITY_INIT;
1641 if (sc->flags & TP_PARITY_INIT) {
1642 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1643 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1646 if (!(sc->flags & QUEUES_BOUND)) {
1648 setup_hw_filters(sc);
1649 sc->flags |= QUEUES_BOUND;
1652 t3_sge_reset_adapter(sc);
1658 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1659 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1660 * during controller_detach, not here.
1663 cxgb_down(struct adapter *sc)
1666 t3_intr_disable(sc);
1670 * if_init for cxgb ports.
1673 cxgb_init(void *arg)
1675 struct port_info *p = arg;
1676 struct adapter *sc = p->adapter;
1679 cxgb_init_locked(p); /* releases adapter lock */
1680 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1684 cxgb_init_locked(struct port_info *p)
1686 struct adapter *sc = p->adapter;
1687 struct ifnet *ifp = p->ifp;
1688 struct cmac *mac = &p->mac;
1689 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1691 ADAPTER_LOCK_ASSERT_OWNED(sc);
1693 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1695 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1704 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1707 * The code that runs during one-time adapter initialization can sleep
1708 * so it's important not to hold any locks across it.
1710 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1718 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1722 if (isset(&sc->open_device_map, p->port_id) &&
1723 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1727 t3_port_intr_enable(sc, p->port_id);
1728 if (!mac->multiport)
1730 cxgb_update_mac_settings(p);
1731 t3_link_start(&p->phy, mac, &p->link_config);
1732 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1733 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1734 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1737 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1738 struct sge_qset *qs = &sc->sge.qs[i];
1739 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1741 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1742 txq->txq_watchdog.c_cpu);
1746 setbit(&sc->open_device_map, p->port_id);
1747 callout_reset(&p->link_check_ch,
1748 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1749 link_check_callout, p);
1754 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1758 wakeup_one(&sc->flags);
1764 cxgb_uninit_locked(struct port_info *p)
1766 struct adapter *sc = p->adapter;
1769 ADAPTER_LOCK_ASSERT_OWNED(sc);
1771 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1772 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1781 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1785 rc = cxgb_uninit_synchronized(p);
1788 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1790 wakeup_one(&sc->flags);
1797 * Called on "ifconfig down", and from port_detach
1800 cxgb_uninit_synchronized(struct port_info *pi)
1802 struct adapter *sc = pi->adapter;
1803 struct ifnet *ifp = pi->ifp;
1806 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1808 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1811 * Clear this port's bit from the open device map, and then drain all
1812 * the tasks that can access/manipulate this port's port_info or ifp.
1813 * We disable this port's interrupts here and so the slow/ext
1814 * interrupt tasks won't be enqueued. The tick task will continue to
1815 * be enqueued every second but the runs after this drain will not see
1816 * this port in the open device map.
1818 * A well behaved task must take open_device_map into account and ignore
1819 * ports that are not open.
1821 clrbit(&sc->open_device_map, pi->port_id);
1822 t3_port_intr_disable(sc, pi->port_id);
1823 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1824 taskqueue_drain(sc->tq, &sc->tick_task);
1826 callout_drain(&pi->link_check_ch);
1827 taskqueue_drain(sc->tq, &pi->link_check_task);
1830 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1832 /* disable pause frames */
1833 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1835 /* Reset RX FIFO HWM */
1836 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1837 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1841 /* Wait for TXFIFO empty */
1842 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1843 F_TXFIFO_EMPTY, 1, 20, 5);
1846 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1848 pi->phy.ops->power_down(&pi->phy, 1);
1852 pi->link_config.link_ok = 0;
1853 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1855 if (sc->open_device_map == 0)
1856 cxgb_down(pi->adapter);
1862 * Mark lro enabled or disabled in all qsets for this port
1865 cxgb_set_lro(struct port_info *p, int enabled)
1868 struct adapter *adp = p->adapter;
1871 for (i = 0; i < p->nqsets; i++) {
1872 q = &adp->sge.qs[p->first_qset + i];
1873 q->lro.enabled = (enabled != 0);
1879 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1881 struct port_info *p = ifp->if_softc;
1882 struct adapter *sc = p->adapter;
1883 struct ifreq *ifr = (struct ifreq *)data;
1884 int flags, error = 0, mtu;
1890 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1898 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1903 cxgb_update_mac_settings(p);
1914 if (ifp->if_flags & IFF_UP) {
1915 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1916 flags = p->if_flags;
1917 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1918 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
1924 cxgb_update_mac_settings(p);
1929 error = cxgb_init_locked(p);
1930 p->if_flags = ifp->if_flags;
1931 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1932 error = cxgb_uninit_locked(p);
1936 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1941 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1945 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1947 cxgb_update_mac_settings(p);
1955 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1959 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1960 if (mask & IFCAP_TXCSUM) {
1961 ifp->if_capenable ^= IFCAP_TXCSUM;
1962 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1964 if (IFCAP_TSO4 & ifp->if_capenable &&
1965 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1966 ifp->if_capenable &= ~IFCAP_TSO4;
1968 "tso4 disabled due to -txcsum.\n");
1971 if (mask & IFCAP_TXCSUM_IPV6) {
1972 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1973 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1975 if (IFCAP_TSO6 & ifp->if_capenable &&
1976 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1977 ifp->if_capenable &= ~IFCAP_TSO6;
1979 "tso6 disabled due to -txcsum6.\n");
1982 if (mask & IFCAP_RXCSUM)
1983 ifp->if_capenable ^= IFCAP_RXCSUM;
1984 if (mask & IFCAP_RXCSUM_IPV6)
1985 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1988 * Note that we leave CSUM_TSO alone (it is always set). The
1989 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1990 * sending a TSO request our way, so it's sufficient to toggle
1993 if (mask & IFCAP_TSO4) {
1994 if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1995 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1996 if_printf(ifp, "enable txcsum first.\n");
2000 ifp->if_capenable ^= IFCAP_TSO4;
2002 if (mask & IFCAP_TSO6) {
2003 if (!(IFCAP_TSO6 & ifp->if_capenable) &&
2004 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2005 if_printf(ifp, "enable txcsum6 first.\n");
2009 ifp->if_capenable ^= IFCAP_TSO6;
2011 if (mask & IFCAP_LRO) {
2012 ifp->if_capenable ^= IFCAP_LRO;
2014 /* Safe to do this even if cxgb_up not called yet */
2015 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2018 if (mask & IFCAP_TOE4) {
2019 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
2021 error = toe_capability(p, enable);
2023 ifp->if_capenable ^= mask;
2026 if (mask & IFCAP_VLAN_HWTAGGING) {
2027 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2028 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2030 cxgb_update_mac_settings(p);
2034 if (mask & IFCAP_VLAN_MTU) {
2035 ifp->if_capenable ^= IFCAP_VLAN_MTU;
2036 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2038 cxgb_update_mac_settings(p);
2042 if (mask & IFCAP_VLAN_HWTSO)
2043 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2044 if (mask & IFCAP_VLAN_HWCSUM)
2045 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2047 #ifdef VLAN_CAPABILITIES
2048 VLAN_CAPABILITIES(ifp);
2054 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2057 error = ether_ioctl(ifp, command, data);
2064 cxgb_media_change(struct ifnet *ifp)
2066 return (EOPNOTSUPP);
2070 * Translates phy->modtype to the correct Ethernet media subtype.
2073 cxgb_ifm_type(int mod)
2076 case phy_modtype_sr:
2077 return (IFM_10G_SR);
2078 case phy_modtype_lr:
2079 return (IFM_10G_LR);
2080 case phy_modtype_lrm:
2081 return (IFM_10G_LRM);
2082 case phy_modtype_twinax:
2083 return (IFM_10G_TWINAX);
2084 case phy_modtype_twinax_long:
2085 return (IFM_10G_TWINAX_LONG);
2086 case phy_modtype_none:
2088 case phy_modtype_unknown:
2089 return (IFM_UNKNOWN);
2092 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2093 return (IFM_UNKNOWN);
2097 * Rebuilds the ifmedia list for this port, and sets the current media.
2100 cxgb_build_medialist(struct port_info *p)
2102 struct cphy *phy = &p->phy;
2103 struct ifmedia *media = &p->media;
2104 int mod = phy->modtype;
2105 int m = IFM_ETHER | IFM_FDX;
2109 ifmedia_removeall(media);
2110 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2113 if (phy->caps & SUPPORTED_10000baseT_Full)
2114 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2116 if (phy->caps & SUPPORTED_1000baseT_Full)
2117 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2119 if (phy->caps & SUPPORTED_100baseT_Full)
2120 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2122 if (phy->caps & SUPPORTED_10baseT_Full)
2123 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2125 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2126 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2128 } else if (phy->caps & SUPPORTED_TP) {
2131 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2132 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2134 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2135 ifmedia_set(media, m | IFM_10G_CX4);
2137 } else if (phy->caps & SUPPORTED_FIBRE &&
2138 phy->caps & SUPPORTED_10000baseT_Full) {
2139 /* 10G optical (but includes SFP+ twinax) */
2141 m |= cxgb_ifm_type(mod);
2142 if (IFM_SUBTYPE(m) == IFM_NONE)
2145 ifmedia_add(media, m, mod, NULL);
2146 ifmedia_set(media, m);
2148 } else if (phy->caps & SUPPORTED_FIBRE &&
2149 phy->caps & SUPPORTED_1000baseT_Full) {
2152 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2153 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2154 ifmedia_set(media, m | IFM_1000_SX);
2157 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2165 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2167 struct port_info *p = ifp->if_softc;
2168 struct ifmedia_entry *cur = p->media.ifm_cur;
2169 int speed = p->link_config.speed;
2171 if (cur->ifm_data != p->phy.modtype) {
2172 cxgb_build_medialist(p);
2173 cur = p->media.ifm_cur;
2176 ifmr->ifm_status = IFM_AVALID;
2177 if (!p->link_config.link_ok)
2180 ifmr->ifm_status |= IFM_ACTIVE;
2183 * active and current will differ iff current media is autoselect. That
2184 * can happen only for copper RJ45.
2186 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2188 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2189 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2191 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2192 if (speed == SPEED_10000)
2193 ifmr->ifm_active |= IFM_10G_T;
2194 else if (speed == SPEED_1000)
2195 ifmr->ifm_active |= IFM_1000_T;
2196 else if (speed == SPEED_100)
2197 ifmr->ifm_active |= IFM_100_TX;
2198 else if (speed == SPEED_10)
2199 ifmr->ifm_active |= IFM_10_T;
2201 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2206 cxgb_get_counter(struct ifnet *ifp, ift_counter c)
2208 struct port_info *pi = ifp->if_softc;
2209 struct adapter *sc = pi->adapter;
2210 struct cmac *mac = &pi->mac;
2211 struct mac_stats *mstats = &mac->stats;
2213 cxgb_refresh_stats(pi);
2216 case IFCOUNTER_IPACKETS:
2217 return (mstats->rx_frames);
2219 case IFCOUNTER_IERRORS:
2220 return (mstats->rx_jabber + mstats->rx_data_errs +
2221 mstats->rx_sequence_errs + mstats->rx_runt +
2222 mstats->rx_too_long + mstats->rx_mac_internal_errs +
2223 mstats->rx_short + mstats->rx_fcs_errs);
2225 case IFCOUNTER_OPACKETS:
2226 return (mstats->tx_frames);
2228 case IFCOUNTER_OERRORS:
2229 return (mstats->tx_excess_collisions + mstats->tx_underrun +
2230 mstats->tx_len_errs + mstats->tx_mac_internal_errs +
2231 mstats->tx_excess_deferral + mstats->tx_fcs_errs);
2233 case IFCOUNTER_COLLISIONS:
2234 return (mstats->tx_total_collisions);
2236 case IFCOUNTER_IBYTES:
2237 return (mstats->rx_octets);
2239 case IFCOUNTER_OBYTES:
2240 return (mstats->tx_octets);
2242 case IFCOUNTER_IMCASTS:
2243 return (mstats->rx_mcast_frames);
2245 case IFCOUNTER_OMCASTS:
2246 return (mstats->tx_mcast_frames);
2248 case IFCOUNTER_IQDROPS:
2249 return (mstats->rx_cong_drops);
2251 case IFCOUNTER_OQDROPS: {
2256 if (sc->flags & FULL_INIT_DONE) {
2257 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
2258 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
2266 return (if_get_counter_default(ifp, c));
2271 cxgb_async_intr(void *data)
2273 adapter_t *sc = data;
2275 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2276 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2277 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2281 link_check_callout(void *arg)
2283 struct port_info *pi = arg;
2284 struct adapter *sc = pi->adapter;
2286 if (!isset(&sc->open_device_map, pi->port_id))
2289 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2293 check_link_status(void *arg, int pending)
2295 struct port_info *pi = arg;
2296 struct adapter *sc = pi->adapter;
2298 if (!isset(&sc->open_device_map, pi->port_id))
2301 t3_link_changed(sc, pi->port_id);
2303 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
2304 pi->link_config.link_ok == 0)
2305 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2309 t3_os_link_intr(struct port_info *pi)
2312 * Schedule a link check in the near future. If the link is flapping
2313 * rapidly we'll keep resetting the callout and delaying the check until
2314 * things stabilize a bit.
2316 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2320 check_t3b2_mac(struct adapter *sc)
2324 if (sc->flags & CXGB_SHUTDOWN)
2327 for_each_port(sc, i) {
2328 struct port_info *p = &sc->port[i];
2331 struct ifnet *ifp = p->ifp;
2334 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2335 !p->link_config.link_ok)
2338 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2339 ("%s: state mismatch (drv_flags %x, device_map %x)",
2340 __func__, ifp->if_drv_flags, sc->open_device_map));
2343 status = t3b2_mac_watchdog_task(&p->mac);
2345 p->mac.stats.num_toggled++;
2346 else if (status == 2) {
2347 struct cmac *mac = &p->mac;
2349 cxgb_update_mac_settings(p);
2350 t3_link_start(&p->phy, mac, &p->link_config);
2351 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2352 t3_port_intr_enable(sc, p->port_id);
2353 p->mac.stats.num_resets++;
2360 cxgb_tick(void *arg)
2362 adapter_t *sc = (adapter_t *)arg;
2364 if (sc->flags & CXGB_SHUTDOWN)
2367 taskqueue_enqueue(sc->tq, &sc->tick_task);
2368 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2372 cxgb_refresh_stats(struct port_info *pi)
2375 const struct timeval interval = {0, 250000}; /* 250ms */
2378 timevalsub(&tv, &interval);
2379 if (timevalcmp(&tv, &pi->last_refreshed, <))
2383 t3_mac_update_stats(&pi->mac);
2385 getmicrotime(&pi->last_refreshed);
2389 cxgb_tick_handler(void *arg, int count)
2391 adapter_t *sc = (adapter_t *)arg;
2392 const struct adapter_params *p = &sc->params;
2394 uint32_t cause, reset;
2396 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2399 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2402 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2404 struct sge_qset *qs = &sc->sge.qs[0];
2407 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2410 for (i = 0; i < SGE_QSETS; i++) {
2412 qs[i].rspq.starved++;
2416 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2418 for (i = 0; i < SGE_QSETS * 2; i++) {
2420 qs[i / 2].fl[i % 2].empty++;
2426 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2427 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2430 for (i = 0; i < sc->params.nports; i++) {
2431 struct port_info *pi = &sc->port[i];
2432 struct cmac *mac = &pi->mac;
2434 if (!isset(&sc->open_device_map, pi->port_id))
2437 cxgb_refresh_stats(pi);
2442 /* Count rx fifo overflows, once per second */
2443 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2445 if (cause & F_RXFIFO_OVERFLOW) {
2446 mac->stats.rx_fifo_ovfl++;
2447 reset |= F_RXFIFO_OVERFLOW;
2449 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2454 touch_bars(device_t dev)
2459 #if !defined(__LP64__) && 0
2462 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2463 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2464 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2465 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2466 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2467 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2472 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2476 u32 aligned_offset, aligned_len, *p;
2477 struct adapter *adapter = pi->adapter;
2480 aligned_offset = offset & ~3;
2481 aligned_len = (len + (offset & 3) + 3) & ~3;
2483 if (aligned_offset != offset || aligned_len != len) {
2484 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2487 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2488 if (!err && aligned_len > 4)
2489 err = t3_seeprom_read(adapter,
2490 aligned_offset + aligned_len - 4,
2491 (u32 *)&buf[aligned_len - 4]);
2494 memcpy(buf + (offset & 3), data, len);
2496 buf = (uint8_t *)(uintptr_t)data;
2498 err = t3_seeprom_wp(adapter, 0);
2502 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2503 err = t3_seeprom_write(adapter, aligned_offset, *p);
2504 aligned_offset += 4;
2508 err = t3_seeprom_wp(adapter, 1);
2511 free(buf, M_DEVBUF);
2517 in_range(int val, int lo, int hi)
2519 return val < 0 || (val <= hi && val >= lo);
2523 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2529 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2535 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2536 int fflag, struct thread *td)
2539 struct port_info *pi = dev->si_drv1;
2540 adapter_t *sc = pi->adapter;
2542 #ifdef PRIV_SUPPORTED
2543 if (priv_check(td, PRIV_DRIVER)) {
2545 printf("user does not have access to privileged ioctls\n");
2551 printf("user does not have access to privileged ioctls\n");
2557 case CHELSIO_GET_MIIREG: {
2559 struct cphy *phy = &pi->phy;
2560 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2562 if (!phy->mdio_read)
2563 return (EOPNOTSUPP);
2565 mmd = mid->phy_id >> 8;
2568 else if (mmd > MDIO_DEV_VEND2)
2571 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2572 mid->reg_num, &val);
2574 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2575 mid->reg_num & 0x1f, &val);
2580 case CHELSIO_SET_MIIREG: {
2581 struct cphy *phy = &pi->phy;
2582 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2584 if (!phy->mdio_write)
2585 return (EOPNOTSUPP);
2587 mmd = mid->phy_id >> 8;
2590 else if (mmd > MDIO_DEV_VEND2)
2593 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2594 mmd, mid->reg_num, mid->val_in);
2596 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2597 mid->reg_num & 0x1f,
2601 case CHELSIO_SETREG: {
2602 struct ch_reg *edata = (struct ch_reg *)data;
2603 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2605 t3_write_reg(sc, edata->addr, edata->val);
2608 case CHELSIO_GETREG: {
2609 struct ch_reg *edata = (struct ch_reg *)data;
2610 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2612 edata->val = t3_read_reg(sc, edata->addr);
2615 case CHELSIO_GET_SGE_CONTEXT: {
2616 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2617 mtx_lock_spin(&sc->sge.reg_lock);
2618 switch (ecntxt->cntxt_type) {
2619 case CNTXT_TYPE_EGRESS:
2620 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2624 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2627 case CNTXT_TYPE_RSP:
2628 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2632 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2639 mtx_unlock_spin(&sc->sge.reg_lock);
2642 case CHELSIO_GET_SGE_DESC: {
2643 struct ch_desc *edesc = (struct ch_desc *)data;
2645 if (edesc->queue_num >= SGE_QSETS * 6)
2647 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2648 edesc->queue_num % 6, edesc->idx, edesc->data);
2654 case CHELSIO_GET_QSET_PARAMS: {
2655 struct qset_params *q;
2656 struct ch_qset_params *t = (struct ch_qset_params *)data;
2657 int q1 = pi->first_qset;
2658 int nqsets = pi->nqsets;
2661 if (t->qset_idx >= nqsets)
2664 i = q1 + t->qset_idx;
2665 q = &sc->params.sge.qset[i];
2666 t->rspq_size = q->rspq_size;
2667 t->txq_size[0] = q->txq_size[0];
2668 t->txq_size[1] = q->txq_size[1];
2669 t->txq_size[2] = q->txq_size[2];
2670 t->fl_size[0] = q->fl_size;
2671 t->fl_size[1] = q->jumbo_size;
2672 t->polling = q->polling;
2674 t->intr_lat = q->coalesce_usecs;
2675 t->cong_thres = q->cong_thres;
2678 if ((sc->flags & FULL_INIT_DONE) == 0)
2680 else if (sc->flags & USING_MSIX)
2681 t->vector = rman_get_start(sc->msix_irq_res[i]);
2683 t->vector = rman_get_start(sc->irq_res);
2687 case CHELSIO_GET_QSET_NUM: {
2688 struct ch_reg *edata = (struct ch_reg *)data;
2689 edata->val = pi->nqsets;
2692 case CHELSIO_LOAD_FW: {
2695 struct ch_mem_range *t = (struct ch_mem_range *)data;
2698 * You're allowed to load a firmware only before FULL_INIT_DONE
2700 * FW_UPTODATE is also set so the rest of the initialization
2701 * will not overwrite what was loaded here. This gives you the
2702 * flexibility to load any firmware (and maybe shoot yourself in
2707 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2712 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2716 error = copyin(t->buf, fw_data, t->len);
2719 error = -t3_load_fw(sc, fw_data, t->len);
2721 if (t3_get_fw_version(sc, &vers) == 0) {
2722 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2723 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2724 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2728 sc->flags |= FW_UPTODATE;
2730 free(fw_data, M_DEVBUF);
2734 case CHELSIO_LOAD_BOOT: {
2736 struct ch_mem_range *t = (struct ch_mem_range *)data;
2738 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2742 error = copyin(t->buf, boot_data, t->len);
2744 error = -t3_load_boot(sc, boot_data, t->len);
2746 free(boot_data, M_DEVBUF);
2749 case CHELSIO_GET_PM: {
2750 struct ch_pm *m = (struct ch_pm *)data;
2751 struct tp_params *p = &sc->params.tp;
2753 if (!is_offload(sc))
2754 return (EOPNOTSUPP);
2756 m->tx_pg_sz = p->tx_pg_size;
2757 m->tx_num_pg = p->tx_num_pgs;
2758 m->rx_pg_sz = p->rx_pg_size;
2759 m->rx_num_pg = p->rx_num_pgs;
2760 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2764 case CHELSIO_SET_PM: {
2765 struct ch_pm *m = (struct ch_pm *)data;
2766 struct tp_params *p = &sc->params.tp;
2768 if (!is_offload(sc))
2769 return (EOPNOTSUPP);
2770 if (sc->flags & FULL_INIT_DONE)
2773 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2774 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2775 return (EINVAL); /* not power of 2 */
2776 if (!(m->rx_pg_sz & 0x14000))
2777 return (EINVAL); /* not 16KB or 64KB */
2778 if (!(m->tx_pg_sz & 0x1554000))
2780 if (m->tx_num_pg == -1)
2781 m->tx_num_pg = p->tx_num_pgs;
2782 if (m->rx_num_pg == -1)
2783 m->rx_num_pg = p->rx_num_pgs;
2784 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2786 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2787 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2790 p->rx_pg_size = m->rx_pg_sz;
2791 p->tx_pg_size = m->tx_pg_sz;
2792 p->rx_num_pgs = m->rx_num_pg;
2793 p->tx_num_pgs = m->tx_num_pg;
2796 case CHELSIO_SETMTUTAB: {
2797 struct ch_mtus *m = (struct ch_mtus *)data;
2800 if (!is_offload(sc))
2801 return (EOPNOTSUPP);
2802 if (offload_running(sc))
2804 if (m->nmtus != NMTUS)
2806 if (m->mtus[0] < 81) /* accommodate SACK */
2810 * MTUs must be in ascending order
2812 for (i = 1; i < NMTUS; ++i)
2813 if (m->mtus[i] < m->mtus[i - 1])
2816 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2819 case CHELSIO_GETMTUTAB: {
2820 struct ch_mtus *m = (struct ch_mtus *)data;
2822 if (!is_offload(sc))
2823 return (EOPNOTSUPP);
2825 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2829 case CHELSIO_GET_MEM: {
2830 struct ch_mem_range *t = (struct ch_mem_range *)data;
2836 * Use these to avoid modifying len/addr in the return
2839 uint32_t len = t->len, addr = t->addr;
2841 if (!is_offload(sc))
2842 return (EOPNOTSUPP);
2843 if (!(sc->flags & FULL_INIT_DONE))
2844 return (EIO); /* need the memory controllers */
2845 if ((addr & 0x7) || (len & 0x7))
2847 if (t->mem_id == MEM_CM)
2849 else if (t->mem_id == MEM_PMRX)
2851 else if (t->mem_id == MEM_PMTX)
2858 * bits 0..9: chip version
2859 * bits 10..15: chip revision
2861 t->version = 3 | (sc->params.rev << 10);
2864 * Read 256 bytes at a time as len can be large and we don't
2865 * want to use huge intermediate buffers.
2867 useraddr = (uint8_t *)t->buf;
2869 unsigned int chunk = min(len, sizeof(buf));
2871 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2874 if (copyout(buf, useraddr, chunk))
2882 case CHELSIO_READ_TCAM_WORD: {
2883 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2885 if (!is_offload(sc))
2886 return (EOPNOTSUPP);
2887 if (!(sc->flags & FULL_INIT_DONE))
2888 return (EIO); /* need MC5 */
2889 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2892 case CHELSIO_SET_TRACE_FILTER: {
2893 struct ch_trace *t = (struct ch_trace *)data;
2894 const struct trace_params *tp;
2896 tp = (const struct trace_params *)&t->sip;
2898 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2901 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2905 case CHELSIO_SET_PKTSCHED: {
2906 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2907 if (sc->open_device_map == 0)
2909 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2913 case CHELSIO_IFCONF_GETREGS: {
2914 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2915 int reglen = cxgb_get_regs_len();
2916 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2920 if (regs->len > reglen)
2922 else if (regs->len < reglen)
2926 cxgb_get_regs(sc, regs, buf);
2927 error = copyout(buf, regs->data, reglen);
2929 free(buf, M_DEVBUF);
2933 case CHELSIO_SET_HW_SCHED: {
2934 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2935 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2937 if ((sc->flags & FULL_INIT_DONE) == 0)
2938 return (EAGAIN); /* need TP to be initialized */
2939 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2940 !in_range(t->channel, 0, 1) ||
2941 !in_range(t->kbps, 0, 10000000) ||
2942 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2943 !in_range(t->flow_ipg, 0,
2944 dack_ticks_to_usec(sc, 0x7ff)))
2948 error = t3_config_sched(sc, t->kbps, t->sched);
2952 if (t->class_ipg >= 0)
2953 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2954 if (t->flow_ipg >= 0) {
2955 t->flow_ipg *= 1000; /* us -> ns */
2956 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2959 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2961 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2962 bit, t->mode ? bit : 0);
2964 if (t->channel >= 0)
2965 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2966 1 << t->sched, t->channel << t->sched);
2969 case CHELSIO_GET_EEPROM: {
2971 struct ch_eeprom *e = (struct ch_eeprom *)data;
2974 if (e->offset & 3 || e->offset >= EEPROMSIZE ||
2975 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
2979 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2983 e->magic = EEPROM_MAGIC;
2984 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2985 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2988 error = copyout(buf + e->offset, e->data, e->len);
2990 free(buf, M_DEVBUF);
2993 case CHELSIO_CLEAR_STATS: {
2994 if (!(sc->flags & FULL_INIT_DONE))
2998 t3_mac_update_stats(&pi->mac);
2999 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
3003 case CHELSIO_GET_UP_LA: {
3004 struct ch_up_la *la = (struct ch_up_la *)data;
3005 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
3009 if (la->bufsize < LA_BUFSIZE)
3013 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
3016 error = copyout(buf, la->data, la->bufsize);
3018 free(buf, M_DEVBUF);
3021 case CHELSIO_GET_UP_IOQS: {
3022 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3023 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3029 if (ioqs->bufsize < IOQS_BUFSIZE)
3033 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3036 v = (uint32_t *)buf;
3038 ioqs->ioq_rx_enable = *v++;
3039 ioqs->ioq_tx_enable = *v++;
3040 ioqs->ioq_rx_status = *v++;
3041 ioqs->ioq_tx_status = *v++;
3043 error = copyout(v, ioqs->data, ioqs->bufsize);
3046 free(buf, M_DEVBUF);
3049 case CHELSIO_SET_FILTER: {
3050 struct ch_filter *f = (struct ch_filter *)data;
3051 struct filter_info *p;
3052 unsigned int nfilters = sc->params.mc5.nfilters;
3054 if (!is_offload(sc))
3055 return (EOPNOTSUPP); /* No TCAM */
3056 if (!(sc->flags & FULL_INIT_DONE))
3057 return (EAGAIN); /* mc5 not setup yet */
3059 return (EBUSY); /* TOE will use TCAM */
3062 if (f->filter_id >= nfilters ||
3063 (f->val.dip && f->mask.dip != 0xffffffff) ||
3064 (f->val.sport && f->mask.sport != 0xffff) ||
3065 (f->val.dport && f->mask.dport != 0xffff) ||
3066 (f->val.vlan && f->mask.vlan != 0xfff) ||
3067 (f->val.vlan_prio &&
3068 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3069 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3070 f->qset >= SGE_QSETS ||
3071 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3074 /* Was allocated with M_WAITOK */
3075 KASSERT(sc->filters, ("filter table NULL\n"));
3077 p = &sc->filters[f->filter_id];
3081 bzero(p, sizeof(*p));
3082 p->sip = f->val.sip;
3083 p->sip_mask = f->mask.sip;
3084 p->dip = f->val.dip;
3085 p->sport = f->val.sport;
3086 p->dport = f->val.dport;
3087 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3088 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3090 p->mac_hit = f->mac_hit;
3091 p->mac_vld = f->mac_addr_idx != 0xffff;
3092 p->mac_idx = f->mac_addr_idx;
3093 p->pkt_type = f->proto;
3094 p->report_filter_id = f->want_filter_id;
3099 error = set_filter(sc, f->filter_id, p);
3104 case CHELSIO_DEL_FILTER: {
3105 struct ch_filter *f = (struct ch_filter *)data;
3106 struct filter_info *p;
3107 unsigned int nfilters = sc->params.mc5.nfilters;
3109 if (!is_offload(sc))
3110 return (EOPNOTSUPP);
3111 if (!(sc->flags & FULL_INIT_DONE))
3113 if (nfilters == 0 || sc->filters == NULL)
3115 if (f->filter_id >= nfilters)
3118 p = &sc->filters[f->filter_id];
3122 return (EFAULT); /* Read "Bad address" as "Bad index" */
3124 bzero(p, sizeof(*p));
3125 p->sip = p->sip_mask = 0xffffffff;
3127 p->vlan_prio = FILTER_NO_VLAN_PRI;
3129 error = set_filter(sc, f->filter_id, p);
3132 case CHELSIO_GET_FILTER: {
3133 struct ch_filter *f = (struct ch_filter *)data;
3134 struct filter_info *p;
3135 unsigned int i, nfilters = sc->params.mc5.nfilters;
3137 if (!is_offload(sc))
3138 return (EOPNOTSUPP);
3139 if (!(sc->flags & FULL_INIT_DONE))
3141 if (nfilters == 0 || sc->filters == NULL)
3144 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3145 for (; i < nfilters; i++) {
3146 p = &sc->filters[i];
3150 bzero(f, sizeof(*f));
3153 f->val.sip = p->sip;
3154 f->mask.sip = p->sip_mask;
3155 f->val.dip = p->dip;
3156 f->mask.dip = p->dip ? 0xffffffff : 0;
3157 f->val.sport = p->sport;
3158 f->mask.sport = p->sport ? 0xffff : 0;
3159 f->val.dport = p->dport;
3160 f->mask.dport = p->dport ? 0xffff : 0;
3161 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3162 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3163 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3165 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3166 0 : FILTER_NO_VLAN_PRI;
3167 f->mac_hit = p->mac_hit;
3168 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3169 f->proto = p->pkt_type;
3170 f->want_filter_id = p->report_filter_id;
3179 f->filter_id = 0xffffffff;
3183 return (EOPNOTSUPP);
3190 static __inline void
3191 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3194 uint32_t *p = (uint32_t *)(buf + start);
3196 for ( ; start <= end; start += sizeof(uint32_t))
3197 *p++ = t3_read_reg(ap, start);
3200 #define T3_REGMAP_SIZE (3 * 1024)
3202 cxgb_get_regs_len(void)
3204 return T3_REGMAP_SIZE;
3208 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3213 * bits 0..9: chip version
3214 * bits 10..15: chip revision
3215 * bit 31: set for PCIe cards
3217 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3220 * We skip the MAC statistics registers because they are clear-on-read.
3221 * Also reading multi-register stats would need to synchronize with the
3222 * periodic mac stats accumulation. Hard to justify the complexity.
3224 memset(buf, 0, cxgb_get_regs_len());
3225 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3226 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3227 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3228 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3229 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3230 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3231 XGM_REG(A_XGM_SERDES_STAT3, 1));
3232 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3233 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3237 alloc_filters(struct adapter *sc)
3239 struct filter_info *p;
3240 unsigned int nfilters = sc->params.mc5.nfilters;
3245 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3248 p = &sc->filters[nfilters - 1];
3250 p->vlan_prio = FILTER_NO_VLAN_PRI;
3251 p->pass = p->rss = p->valid = p->locked = 1;
3257 setup_hw_filters(struct adapter *sc)
3260 unsigned int nfilters = sc->params.mc5.nfilters;
3265 t3_enable_filters(sc);
3267 for (i = rc = 0; i < nfilters && !rc; i++) {
3268 if (sc->filters[i].locked)
3269 rc = set_filter(sc, i, &sc->filters[i]);
3276 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3280 struct ulp_txpkt *txpkt;
3281 struct work_request_hdr *wr;
3282 struct cpl_pass_open_req *oreq;
3283 struct cpl_set_tcb_field *sreq;
3285 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3286 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3288 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3289 sc->params.mc5.nfilters;
3291 m = m_gethdr(M_WAITOK, MT_DATA);
3292 m->m_len = m->m_pkthdr.len = len;
3293 bzero(mtod(m, char *), len);
3295 wr = mtod(m, struct work_request_hdr *);
3296 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3298 oreq = (struct cpl_pass_open_req *)(wr + 1);
3299 txpkt = (struct ulp_txpkt *)oreq;
3300 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3301 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3302 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3303 oreq->local_port = htons(f->dport);
3304 oreq->peer_port = htons(f->sport);
3305 oreq->local_ip = htonl(f->dip);
3306 oreq->peer_ip = htonl(f->sip);
3307 oreq->peer_netmask = htonl(f->sip_mask);
3309 oreq->opt0l = htonl(F_NO_OFFLOAD);
3310 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3311 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3312 V_VLAN_PRI(f->vlan_prio >> 1) |
3313 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3314 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3315 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3317 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3318 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3319 (f->report_filter_id << 15) | (1 << 23) |
3320 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3321 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3324 if (f->pass && !f->rss) {
3325 len = sizeof(*sreq);
3326 m = m_gethdr(M_WAITOK, MT_DATA);
3327 m->m_len = m->m_pkthdr.len = len;
3328 bzero(mtod(m, char *), len);
3329 sreq = mtod(m, struct cpl_set_tcb_field *);
3330 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3331 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3332 (u64)sc->rrss_map[f->qset] << 19);
3339 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3340 unsigned int word, u64 mask, u64 val)
3342 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3343 req->reply = V_NO_REPLY(1);
3345 req->word = htons(word);
3346 req->mask = htobe64(mask);
3347 req->val = htobe64(val);
3351 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3352 unsigned int word, u64 mask, u64 val)
3354 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3356 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3357 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3358 mk_set_tcb_field(req, tid, word, mask, val);
3362 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3366 mtx_lock(&t3_list_lock);
3367 SLIST_FOREACH(sc, &t3_list, link) {
3369 * func should not make any assumptions about what state sc is
3370 * in - the only guarantee is that sc->sc_lock is a valid lock.
3374 mtx_unlock(&t3_list_lock);
3379 toe_capability(struct port_info *pi, int enable)
3382 struct adapter *sc = pi->adapter;
3384 ADAPTER_LOCK_ASSERT_OWNED(sc);
3386 if (!is_offload(sc))
3390 if (!(sc->flags & FULL_INIT_DONE)) {
3392 "You must enable a cxgb interface first\n");
3396 if (isset(&sc->offload_map, pi->port_id))
3399 if (!(sc->flags & TOM_INIT_DONE)) {
3400 rc = t3_activate_uld(sc, ULD_TOM);
3403 "You must kldload t3_tom.ko before trying "
3404 "to enable TOE on a cxgb interface.\n");
3408 KASSERT(sc->tom_softc != NULL,
3409 ("%s: TOM activated but softc NULL", __func__));
3410 KASSERT(sc->flags & TOM_INIT_DONE,
3411 ("%s: TOM activated but flag not set", __func__));
3414 setbit(&sc->offload_map, pi->port_id);
3417 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3418 * enabled on any port. Need to figure out how to enable,
3419 * disable, load, and unload iWARP cleanly.
3421 if (!isset(&sc->offload_map, MAX_NPORTS) &&
3422 t3_activate_uld(sc, ULD_IWARP) == 0)
3423 setbit(&sc->offload_map, MAX_NPORTS);
3425 if (!isset(&sc->offload_map, pi->port_id))
3428 KASSERT(sc->flags & TOM_INIT_DONE,
3429 ("%s: TOM never initialized?", __func__));
3430 clrbit(&sc->offload_map, pi->port_id);
3437 * Add an upper layer driver to the global list.
3440 t3_register_uld(struct uld_info *ui)
3445 mtx_lock(&t3_uld_list_lock);
3446 SLIST_FOREACH(u, &t3_uld_list, link) {
3447 if (u->uld_id == ui->uld_id) {
3453 SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3456 mtx_unlock(&t3_uld_list_lock);
3461 t3_unregister_uld(struct uld_info *ui)
3466 mtx_lock(&t3_uld_list_lock);
3468 SLIST_FOREACH(u, &t3_uld_list, link) {
3470 if (ui->refcount > 0) {
3475 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3481 mtx_unlock(&t3_uld_list_lock);
3486 t3_activate_uld(struct adapter *sc, int id)
3489 struct uld_info *ui;
3491 mtx_lock(&t3_uld_list_lock);
3493 SLIST_FOREACH(ui, &t3_uld_list, link) {
3494 if (ui->uld_id == id) {
3495 rc = ui->activate(sc);
3502 mtx_unlock(&t3_uld_list_lock);
3508 t3_deactivate_uld(struct adapter *sc, int id)
3511 struct uld_info *ui;
3513 mtx_lock(&t3_uld_list_lock);
3515 SLIST_FOREACH(ui, &t3_uld_list, link) {
3516 if (ui->uld_id == id) {
3517 rc = ui->deactivate(sc);
3524 mtx_unlock(&t3_uld_list_lock);
3530 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3538 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3540 uintptr_t *loc, new;
3542 if (opcode >= NUM_CPL_HANDLERS)
3545 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3546 loc = (uintptr_t *) &sc->cpl_handler[opcode];
3547 atomic_store_rel_ptr(loc, new);
3554 cxgbc_mod_event(module_t mod, int cmd, void *arg)
3560 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3561 SLIST_INIT(&t3_list);
3563 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3564 SLIST_INIT(&t3_uld_list);
3570 mtx_lock(&t3_uld_list_lock);
3571 if (!SLIST_EMPTY(&t3_uld_list)) {
3573 mtx_unlock(&t3_uld_list_lock);
3576 mtx_unlock(&t3_uld_list_lock);
3577 mtx_destroy(&t3_uld_list_lock);
3579 mtx_lock(&t3_list_lock);
3580 if (!SLIST_EMPTY(&t3_list)) {
3582 mtx_unlock(&t3_list_lock);
3585 mtx_unlock(&t3_list_lock);
3586 mtx_destroy(&t3_list_lock);
3595 cxgb_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
3597 struct port_info *pi;
3600 pi = if_getsoftc(ifp);
3603 *nrxr = adap->nqsets;
3604 *ncl = adap->sge.qs[0].fl[1].size;
3605 *clsize = adap->sge.qs[0].fl[1].buf_size;
3606 ADAPTER_UNLOCK(adap);
3610 cxgb_netdump_event(struct ifnet *ifp, enum netdump_ev event)
3612 struct port_info *pi;
3613 struct sge_qset *qs;
3616 pi = if_getsoftc(ifp);
3617 if (event == NETDUMP_START)
3618 for (i = 0; i < pi->adapter->nqsets; i++) {
3619 qs = &pi->adapter->sge.qs[i];
3621 /* Need to reinit after netdump_mbuf_dump(). */
3622 qs->fl[0].zone = zone_pack;
3623 qs->fl[1].zone = zone_clust;
3624 qs->lro.enabled = 0;
3629 cxgb_netdump_transmit(struct ifnet *ifp, struct mbuf *m)
3631 struct port_info *pi;
3632 struct sge_qset *qs;
3634 pi = if_getsoftc(ifp);
3635 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
3639 qs = &pi->adapter->sge.qs[pi->first_qset];
3640 return (cxgb_netdump_encap(qs, &m));
3644 cxgb_netdump_poll(struct ifnet *ifp, int count)
3646 struct port_info *pi;
3650 pi = if_getsoftc(ifp);
3651 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
3655 for (i = 0; i < adap->nqsets; i++)
3656 (void)cxgb_netdump_poll_rx(adap, &adap->sge.qs[i]);
3657 (void)cxgb_netdump_poll_tx(&adap->sge.qs[pi->first_qset]);
3660 #endif /* NETDUMP */