1 /**************************************************************************
3 Copyright (c) 2007-2009, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
39 #include <sys/module.h>
40 #include <sys/pciio.h>
42 #include <machine/bus.h>
43 #include <machine/resource.h>
46 #include <sys/ioccom.h>
48 #include <sys/linker.h>
49 #include <sys/firmware.h>
50 #include <sys/socket.h>
51 #include <sys/sockio.h>
53 #include <sys/sysctl.h>
54 #include <sys/syslog.h>
55 #include <sys/queue.h>
56 #include <sys/taskqueue.h>
60 #include <net/ethernet.h>
62 #include <net/if_var.h>
63 #include <net/if_arp.h>
64 #include <net/if_dl.h>
65 #include <net/if_media.h>
66 #include <net/if_types.h>
67 #include <net/if_vlan_var.h>
69 #include <netinet/in_systm.h>
70 #include <netinet/in.h>
71 #include <netinet/if_ether.h>
72 #include <netinet/ip.h>
73 #include <netinet/ip.h>
74 #include <netinet/tcp.h>
75 #include <netinet/udp.h>
77 #include <dev/pci/pcireg.h>
78 #include <dev/pci/pcivar.h>
79 #include <dev/pci/pci_private.h>
81 #include <cxgb_include.h>
87 static int cxgb_setup_interrupts(adapter_t *);
88 static void cxgb_teardown_interrupts(adapter_t *);
89 static void cxgb_init(void *);
90 static int cxgb_init_locked(struct port_info *);
91 static int cxgb_uninit_locked(struct port_info *);
92 static int cxgb_uninit_synchronized(struct port_info *);
93 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
94 static int cxgb_media_change(struct ifnet *);
95 static int cxgb_ifm_type(int);
96 static void cxgb_build_medialist(struct port_info *);
97 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
98 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
99 static int setup_sge_qsets(adapter_t *);
100 static void cxgb_async_intr(void *);
101 static void cxgb_tick_handler(void *, int);
102 static void cxgb_tick(void *);
103 static void link_check_callout(void *);
104 static void check_link_status(void *, int);
105 static void setup_rss(adapter_t *sc);
106 static int alloc_filters(struct adapter *);
107 static int setup_hw_filters(struct adapter *);
108 static int set_filter(struct adapter *, int, const struct filter_info *);
109 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
110 unsigned int, u64, u64);
111 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
112 unsigned int, u64, u64);
114 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
117 /* Attachment glue for the PCI controller end of the device. Each port of
118 * the device is attached separately, as defined later.
120 static int cxgb_controller_probe(device_t);
121 static int cxgb_controller_attach(device_t);
122 static int cxgb_controller_detach(device_t);
123 static void cxgb_free(struct adapter *);
124 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
126 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
127 static int cxgb_get_regs_len(void);
128 static void touch_bars(device_t dev);
129 static void cxgb_update_mac_settings(struct port_info *p);
131 static int toe_capability(struct port_info *, int);
134 static device_method_t cxgb_controller_methods[] = {
135 DEVMETHOD(device_probe, cxgb_controller_probe),
136 DEVMETHOD(device_attach, cxgb_controller_attach),
137 DEVMETHOD(device_detach, cxgb_controller_detach),
142 static driver_t cxgb_controller_driver = {
144 cxgb_controller_methods,
145 sizeof(struct adapter)
148 static int cxgbc_mod_event(module_t, int, void *);
149 static devclass_t cxgb_controller_devclass;
150 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
152 MODULE_VERSION(cxgbc, 1);
153 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
156 * Attachment glue for the ports. Attachment is done directly to the
159 static int cxgb_port_probe(device_t);
160 static int cxgb_port_attach(device_t);
161 static int cxgb_port_detach(device_t);
163 static device_method_t cxgb_port_methods[] = {
164 DEVMETHOD(device_probe, cxgb_port_probe),
165 DEVMETHOD(device_attach, cxgb_port_attach),
166 DEVMETHOD(device_detach, cxgb_port_detach),
170 static driver_t cxgb_port_driver = {
176 static d_ioctl_t cxgb_extension_ioctl;
177 static d_open_t cxgb_extension_open;
178 static d_close_t cxgb_extension_close;
180 static struct cdevsw cxgb_cdevsw = {
181 .d_version = D_VERSION,
183 .d_open = cxgb_extension_open,
184 .d_close = cxgb_extension_close,
185 .d_ioctl = cxgb_extension_ioctl,
189 static devclass_t cxgb_port_devclass;
190 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
191 MODULE_VERSION(cxgb, 1);
193 static struct mtx t3_list_lock;
194 static SLIST_HEAD(, adapter) t3_list;
196 static struct mtx t3_uld_list_lock;
197 static SLIST_HEAD(, uld_info) t3_uld_list;
201 * The driver uses the best interrupt scheme available on a platform in the
202 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
203 * of these schemes the driver may consider as follows:
205 * msi = 2: choose from among all three options
206 * msi = 1 : only consider MSI and pin interrupts
207 * msi = 0: force pin interrupts
209 static int msi_allowed = 2;
211 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
212 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
213 "MSI-X, MSI, INTx selector");
216 * The driver uses an auto-queue algorithm by default.
217 * To disable it and force a single queue-set per port, use multiq = 0
219 static int multiq = 1;
220 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
221 "use min(ncpus/ports, 8) queue-sets per port");
224 * By default the driver will not update the firmware unless
225 * it was compiled against a newer version
228 static int force_fw_update = 0;
229 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
230 "update firmware even if up to date");
232 int cxgb_use_16k_clusters = -1;
233 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
234 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
236 static int nfilters = -1;
237 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
238 &nfilters, 0, "max number of entries in the filter table");
241 MAX_TXQ_ENTRIES = 16384,
242 MAX_CTRL_TXQ_ENTRIES = 1024,
243 MAX_RSPQ_ENTRIES = 16384,
244 MAX_RX_BUFFERS = 16384,
245 MAX_RX_JUMBO_BUFFERS = 16384,
247 MIN_CTRL_TXQ_ENTRIES = 4,
248 MIN_RSPQ_ENTRIES = 32,
250 MIN_FL_JUMBO_ENTRIES = 32
265 u32 report_filter_id:1;
273 enum { FILTER_NO_VLAN_PRI = 7 };
275 #define EEPROM_MAGIC 0x38E2F10C
277 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
279 /* Table for probing the cards. The desc field isn't actually used */
285 } cxgb_identifiers[] = {
286 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
287 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
288 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
289 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
290 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
291 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
292 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
293 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
294 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
295 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
296 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
297 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
298 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
299 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
303 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
307 t3rev2char(struct adapter *adapter)
311 switch(adapter->params.rev) {
326 static struct cxgb_ident *
327 cxgb_get_ident(device_t dev)
329 struct cxgb_ident *id;
331 for (id = cxgb_identifiers; id->desc != NULL; id++) {
332 if ((id->vendor == pci_get_vendor(dev)) &&
333 (id->device == pci_get_device(dev))) {
340 static const struct adapter_info *
341 cxgb_get_adapter_info(device_t dev)
343 struct cxgb_ident *id;
344 const struct adapter_info *ai;
346 id = cxgb_get_ident(dev);
350 ai = t3_get_adapter_info(id->index);
356 cxgb_controller_probe(device_t dev)
358 const struct adapter_info *ai;
359 char *ports, buf[80];
362 ai = cxgb_get_adapter_info(dev);
366 nports = ai->nports0 + ai->nports1;
372 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
373 device_set_desc_copy(dev, buf);
374 return (BUS_PROBE_DEFAULT);
377 #define FW_FNAME "cxgb_t3fw"
378 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
379 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
382 upgrade_fw(adapter_t *sc)
384 const struct firmware *fw;
388 if ((fw = firmware_get(FW_FNAME)) == NULL) {
389 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
392 device_printf(sc->dev, "installing firmware on card\n");
393 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
396 device_printf(sc->dev, "failed to install firmware: %d\n",
399 t3_get_fw_version(sc, &vers);
400 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
401 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
402 G_FW_VERSION_MICRO(vers));
405 firmware_put(fw, FIRMWARE_UNLOAD);
411 * The cxgb_controller_attach function is responsible for the initial
412 * bringup of the device. Its responsibilities include:
414 * 1. Determine if the device supports MSI or MSI-X.
415 * 2. Allocate bus resources so that we can access the Base Address Register
416 * 3. Create and initialize mutexes for the controller and its control
417 * logic such as SGE and MDIO.
418 * 4. Call hardware specific setup routine for the adapter as a whole.
419 * 5. Allocate the BAR for doing MSI-X.
420 * 6. Setup the line interrupt iff MSI-X is not supported.
421 * 7. Create the driver's taskq.
422 * 8. Start one task queue service thread.
423 * 9. Check if the firmware and SRAM are up-to-date. They will be
424 * auto-updated later (before FULL_INIT_DONE), if required.
425 * 10. Create a child device for each MAC (port)
426 * 11. Initialize T3 private state.
427 * 12. Trigger the LED
428 * 13. Setup offload iff supported.
429 * 14. Reset/restart the tick callout.
432 * NOTE: Any modification or deviation from this list MUST be reflected in
433 * the above comment. Failure to do so will result in problems on various
434 * error conditions including link flapping.
437 cxgb_controller_attach(device_t dev)
440 const struct adapter_info *ai;
448 sc = device_get_softc(dev);
451 ai = cxgb_get_adapter_info(dev);
453 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
454 device_get_unit(dev));
455 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
457 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
458 device_get_unit(dev));
459 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
460 device_get_unit(dev));
461 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
462 device_get_unit(dev));
464 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
465 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
466 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
468 mtx_lock(&t3_list_lock);
469 SLIST_INSERT_HEAD(&t3_list, sc, link);
470 mtx_unlock(&t3_list_lock);
472 /* find the PCIe link width and set max read request to 4KB*/
473 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) {
476 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
477 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
478 if (sc->link_width < 8 &&
479 (ai->caps & SUPPORTED_10000baseT_Full)) {
480 device_printf(sc->dev,
481 "PCIe x%d Link, expect reduced performance\n",
485 pci_set_max_read_req(dev, 4096);
489 pci_enable_busmaster(dev);
491 * Allocate the registers and make them available to the driver.
492 * The registers that we care about for NIC mode are in BAR 0
494 sc->regs_rid = PCIR_BAR(0);
495 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
496 &sc->regs_rid, RF_ACTIVE)) == NULL) {
497 device_printf(dev, "Cannot allocate BAR region 0\n");
502 sc->bt = rman_get_bustag(sc->regs_res);
503 sc->bh = rman_get_bushandle(sc->regs_res);
504 sc->mmio_len = rman_get_size(sc->regs_res);
506 for (i = 0; i < MAX_NPORTS; i++)
507 sc->port[i].adapter = sc;
509 if (t3_prep_adapter(sc, ai, 1) < 0) {
510 printf("prep adapter failed\n");
515 sc->udbs_rid = PCIR_BAR(2);
517 if (is_offload(sc) &&
518 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
519 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
520 device_printf(dev, "Cannot allocate BAR region 1\n");
525 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
526 * enough messages for the queue sets. If that fails, try falling
527 * back to MSI. If that fails, then try falling back to the legacy
528 * interrupt pin model.
530 sc->msix_regs_rid = 0x20;
531 if ((msi_allowed >= 2) &&
532 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
533 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
536 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
537 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
539 if (pci_msix_count(dev) == 0 ||
540 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
541 sc->msi_count != msi_needed) {
542 device_printf(dev, "alloc msix failed - "
543 "msi_count=%d, msi_needed=%d, err=%d; "
544 "will try MSI\n", sc->msi_count,
548 pci_release_msi(dev);
549 bus_release_resource(dev, SYS_RES_MEMORY,
550 sc->msix_regs_rid, sc->msix_regs_res);
551 sc->msix_regs_res = NULL;
553 sc->flags |= USING_MSIX;
554 sc->cxgb_intr = cxgb_async_intr;
556 "using MSI-X interrupts (%u vectors)\n",
561 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
563 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
564 device_printf(dev, "alloc msi failed - "
565 "err=%d; will try INTx\n", error);
568 pci_release_msi(dev);
570 sc->flags |= USING_MSI;
571 sc->cxgb_intr = t3_intr_msi;
572 device_printf(dev, "using MSI interrupts\n");
575 if (sc->msi_count == 0) {
576 device_printf(dev, "using line interrupts\n");
577 sc->cxgb_intr = t3b_intr;
580 /* Create a private taskqueue thread for handling driver events */
581 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
582 taskqueue_thread_enqueue, &sc->tq);
583 if (sc->tq == NULL) {
584 device_printf(dev, "failed to allocate controller task queue\n");
588 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
589 device_get_nameunit(dev));
590 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
593 /* Create a periodic callout for checking adapter status */
594 callout_init(&sc->cxgb_tick_ch, 1);
596 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
598 * Warn user that a firmware update will be attempted in init.
600 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
601 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
602 sc->flags &= ~FW_UPTODATE;
604 sc->flags |= FW_UPTODATE;
607 if (t3_check_tpsram_version(sc) < 0) {
609 * Warn user that a firmware update will be attempted in init.
611 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
612 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
613 sc->flags &= ~TPS_UPTODATE;
615 sc->flags |= TPS_UPTODATE;
619 * Create a child device for each MAC. The ethernet attachment
620 * will be done in these children.
622 for (i = 0; i < (sc)->params.nports; i++) {
623 struct port_info *pi;
625 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
626 device_printf(dev, "failed to add child port\n");
632 pi->nqsets = port_qsets;
633 pi->first_qset = i*port_qsets;
635 pi->tx_chan = i >= ai->nports0;
636 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
637 sc->rxpkt_map[pi->txpkt_intf] = i;
638 sc->port[i].tx_chan = i >= ai->nports0;
639 sc->portdev[i] = child;
640 device_set_softc(child, pi);
642 if ((error = bus_generic_attach(dev)) != 0)
645 /* initialize sge private state */
646 t3_sge_init_adapter(sc);
650 error = t3_get_fw_version(sc, &vers);
654 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
655 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
656 G_FW_VERSION_MICRO(vers));
658 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
659 ai->desc, is_offload(sc) ? "R" : "",
660 sc->params.vpd.ec, sc->params.vpd.sn);
661 device_set_desc_copy(dev, buf);
663 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
664 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
665 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
667 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
668 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
669 t3_add_attach_sysctls(sc);
672 for (i = 0; i < NUM_CPL_HANDLERS; i++)
673 sc->cpl_handler[i] = cpl_not_handled;
677 error = cxgb_setup_interrupts(sc);
686 * The cxgb_controller_detach routine is called with the device is
687 * unloaded from the system.
691 cxgb_controller_detach(device_t dev)
695 sc = device_get_softc(dev);
703 * The cxgb_free() is called by the cxgb_controller_detach() routine
704 * to tear down the structures that were built up in
705 * cxgb_controller_attach(), and should be the final piece of work
706 * done when fully unloading the driver.
709 * 1. Shutting down the threads started by the cxgb_controller_attach()
711 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
712 * 3. Detaching all of the port devices created during the
713 * cxgb_controller_attach() routine.
714 * 4. Removing the device children created via cxgb_controller_attach().
715 * 5. Releasing PCI resources associated with the device.
716 * 6. Turning off the offload support, iff it was turned on.
717 * 7. Destroying the mutexes created in cxgb_controller_attach().
721 cxgb_free(struct adapter *sc)
726 sc->flags |= CXGB_SHUTDOWN;
730 * Make sure all child devices are gone.
732 bus_generic_detach(sc->dev);
733 for (i = 0; i < (sc)->params.nports; i++) {
734 if (sc->portdev[i] &&
735 device_delete_child(sc->dev, sc->portdev[i]) != 0)
736 device_printf(sc->dev, "failed to delete child port\n");
737 nqsets += sc->port[i].nqsets;
741 * At this point, it is as if cxgb_port_detach has run on all ports, and
742 * cxgb_down has run on the adapter. All interrupts have been silenced,
743 * all open devices have been closed.
745 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
746 __func__, sc->open_device_map));
747 for (i = 0; i < sc->params.nports; i++) {
748 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
753 * Finish off the adapter's callouts.
755 callout_drain(&sc->cxgb_tick_ch);
756 callout_drain(&sc->sge_timer_ch);
759 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
760 * sysctls are cleaned up by the kernel linker.
762 if (sc->flags & FULL_INIT_DONE) {
763 t3_free_sge_resources(sc, nqsets);
764 sc->flags &= ~FULL_INIT_DONE;
768 * Release all interrupt resources.
770 cxgb_teardown_interrupts(sc);
771 if (sc->flags & (USING_MSI | USING_MSIX)) {
772 device_printf(sc->dev, "releasing msi message(s)\n");
773 pci_release_msi(sc->dev);
775 device_printf(sc->dev, "no msi message to release\n");
778 if (sc->msix_regs_res != NULL) {
779 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
784 * Free the adapter's taskqueue.
786 if (sc->tq != NULL) {
787 taskqueue_free(sc->tq);
791 free(sc->filters, M_DEVBUF);
794 if (sc->udbs_res != NULL)
795 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
798 if (sc->regs_res != NULL)
799 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
802 MTX_DESTROY(&sc->mdio_lock);
803 MTX_DESTROY(&sc->sge.reg_lock);
804 MTX_DESTROY(&sc->elmer_lock);
805 mtx_lock(&t3_list_lock);
806 SLIST_REMOVE(&t3_list, sc, adapter, link);
807 mtx_unlock(&t3_list_lock);
808 ADAPTER_LOCK_DEINIT(sc);
812 * setup_sge_qsets - configure SGE Tx/Rx/response queues
813 * @sc: the controller softc
815 * Determines how many sets of SGE queues to use and initializes them.
816 * We support multiple queue sets per port if we have MSI-X, otherwise
817 * just one queue set per port.
820 setup_sge_qsets(adapter_t *sc)
822 int i, j, err, irq_idx = 0, qset_idx = 0;
823 u_int ntxq = SGE_TXQ_PER_SET;
825 if ((err = t3_sge_alloc(sc)) != 0) {
826 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
830 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
833 for (i = 0; i < (sc)->params.nports; i++) {
834 struct port_info *pi = &sc->port[i];
836 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
837 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
838 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
839 &sc->params.sge.qset[qset_idx], ntxq, pi);
841 t3_free_sge_resources(sc, qset_idx);
842 device_printf(sc->dev,
843 "t3_sge_alloc_qset failed with %d\n", err);
853 cxgb_teardown_interrupts(adapter_t *sc)
857 for (i = 0; i < SGE_QSETS; i++) {
858 if (sc->msix_intr_tag[i] == NULL) {
860 /* Should have been setup fully or not at all */
861 KASSERT(sc->msix_irq_res[i] == NULL &&
862 sc->msix_irq_rid[i] == 0,
863 ("%s: half-done interrupt (%d).", __func__, i));
868 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
869 sc->msix_intr_tag[i]);
870 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
871 sc->msix_irq_res[i]);
873 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
874 sc->msix_irq_rid[i] = 0;
878 KASSERT(sc->irq_res != NULL,
879 ("%s: half-done interrupt.", __func__));
881 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
882 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
885 sc->irq_res = sc->intr_tag = NULL;
891 cxgb_setup_interrupts(adapter_t *sc)
893 struct resource *res;
895 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
897 sc->irq_rid = intr_flag ? 1 : 0;
898 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
899 RF_SHAREABLE | RF_ACTIVE);
900 if (sc->irq_res == NULL) {
901 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
902 intr_flag, sc->irq_rid);
906 err = bus_setup_intr(sc->dev, sc->irq_res,
907 INTR_MPSAFE | INTR_TYPE_NET, NULL,
908 sc->cxgb_intr, sc, &sc->intr_tag);
911 device_printf(sc->dev,
912 "Cannot set up interrupt (%x, %u, %d)\n",
913 intr_flag, sc->irq_rid, err);
914 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
916 sc->irq_res = sc->intr_tag = NULL;
921 /* That's all for INTx or MSI */
922 if (!(intr_flag & USING_MSIX) || err)
925 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
926 for (i = 0; i < sc->msi_count - 1; i++) {
928 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
929 RF_SHAREABLE | RF_ACTIVE);
931 device_printf(sc->dev, "Cannot allocate interrupt "
932 "for message %d\n", rid);
937 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
938 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
940 device_printf(sc->dev, "Cannot set up interrupt "
941 "for message %d (%d)\n", rid, err);
942 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
946 sc->msix_irq_rid[i] = rid;
947 sc->msix_irq_res[i] = res;
948 sc->msix_intr_tag[i] = tag;
949 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
953 cxgb_teardown_interrupts(sc);
960 cxgb_port_probe(device_t dev)
966 p = device_get_softc(dev);
968 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
969 device_set_desc_copy(dev, buf);
975 cxgb_makedev(struct port_info *pi)
978 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
979 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
981 if (pi->port_cdev == NULL)
984 pi->port_cdev->si_drv1 = (void *)pi;
989 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
990 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
991 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
992 #define CXGB_CAP_ENABLE CXGB_CAP
995 cxgb_port_attach(device_t dev)
1002 p = device_get_softc(dev);
1004 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1005 device_get_unit(device_get_parent(dev)), p->port_id);
1006 PORT_LOCK_INIT(p, p->lockbuf);
1008 callout_init(&p->link_check_ch, 1);
1009 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1011 /* Allocate an ifnet object and set it up */
1012 ifp = p->ifp = if_alloc(IFT_ETHER);
1014 device_printf(dev, "Cannot allocate ifnet\n");
1018 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1019 ifp->if_init = cxgb_init;
1021 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1022 ifp->if_ioctl = cxgb_ioctl;
1023 ifp->if_transmit = cxgb_transmit;
1024 ifp->if_qflush = cxgb_qflush;
1025 ifp->if_get_counter = cxgb_get_counter;
1027 ifp->if_capabilities = CXGB_CAP;
1030 ifp->if_capabilities |= IFCAP_TOE4;
1032 ifp->if_capenable = CXGB_CAP_ENABLE;
1033 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1034 CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1037 * Disable TSO on 4-port - it isn't supported by the firmware.
1039 if (sc->params.nports > 2) {
1040 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1041 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1042 ifp->if_hwassist &= ~CSUM_TSO;
1045 ether_ifattach(ifp, p->hw_addr);
1047 #ifdef DEFAULT_JUMBO
1048 if (sc->params.nports <= 2)
1049 ifp->if_mtu = ETHERMTU_JUMBO;
1051 if ((err = cxgb_makedev(p)) != 0) {
1052 printf("makedev failed %d\n", err);
1056 /* Create a list of media supported by this port */
1057 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1059 cxgb_build_medialist(p);
1061 t3_sge_init_port(p);
1067 * cxgb_port_detach() is called via the device_detach methods when
1068 * cxgb_free() calls the bus_generic_detach. It is responsible for
1069 * removing the device from the view of the kernel, i.e. from all
1070 * interfaces lists etc. This routine is only called when the driver is
1071 * being unloaded, not when the link goes down.
1074 cxgb_port_detach(device_t dev)
1076 struct port_info *p;
1080 p = device_get_softc(dev);
1083 /* Tell cxgb_ioctl and if_init that the port is going away */
1088 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1092 if (p->port_cdev != NULL)
1093 destroy_dev(p->port_cdev);
1095 cxgb_uninit_synchronized(p);
1096 ether_ifdetach(p->ifp);
1098 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1099 struct sge_qset *qs = &sc->sge.qs[i];
1100 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1102 callout_drain(&txq->txq_watchdog);
1103 callout_drain(&txq->txq_timer);
1106 PORT_LOCK_DEINIT(p);
1112 wakeup_one(&sc->flags);
1118 t3_fatal_err(struct adapter *sc)
1122 if (sc->flags & FULL_INIT_DONE) {
1124 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1125 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1126 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1127 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1128 t3_intr_disable(sc);
1130 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1131 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1132 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1133 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1137 t3_os_find_pci_capability(adapter_t *sc, int cap)
1140 struct pci_devinfo *dinfo;
1146 dinfo = device_get_ivars(dev);
1149 status = pci_read_config(dev, PCIR_STATUS, 2);
1150 if (!(status & PCIM_STATUS_CAPPRESENT))
1153 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1159 ptr = PCIR_CAP_PTR_2;
1165 ptr = pci_read_config(dev, ptr, 1);
1168 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1170 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1177 t3_os_pci_save_state(struct adapter *sc)
1180 struct pci_devinfo *dinfo;
1183 dinfo = device_get_ivars(dev);
1185 pci_cfg_save(dev, dinfo, 0);
1190 t3_os_pci_restore_state(struct adapter *sc)
1193 struct pci_devinfo *dinfo;
1196 dinfo = device_get_ivars(dev);
1198 pci_cfg_restore(dev, dinfo);
1203 * t3_os_link_changed - handle link status changes
1204 * @sc: the adapter associated with the link change
1205 * @port_id: the port index whose link status has changed
1206 * @link_status: the new status of the link
1207 * @speed: the new speed setting
1208 * @duplex: the new duplex setting
1209 * @fc: the new flow-control setting
1211 * This is the OS-dependent handler for link status changes. The OS
1212 * neutral handler takes care of most of the processing for these events,
1213 * then calls this handler for any OS-specific processing.
1216 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1217 int duplex, int fc, int mac_was_reset)
1219 struct port_info *pi = &adapter->port[port_id];
1220 struct ifnet *ifp = pi->ifp;
1222 /* no race with detach, so ifp should always be good */
1223 KASSERT(ifp, ("%s: if detached.", __func__));
1225 /* Reapply mac settings if they were lost due to a reset */
1226 if (mac_was_reset) {
1228 cxgb_update_mac_settings(pi);
1233 ifp->if_baudrate = IF_Mbps(speed);
1234 if_link_state_change(ifp, LINK_STATE_UP);
1236 if_link_state_change(ifp, LINK_STATE_DOWN);
1240 * t3_os_phymod_changed - handle PHY module changes
1241 * @phy: the PHY reporting the module change
1242 * @mod_type: new module type
1244 * This is the OS-dependent handler for PHY module changes. It is
1245 * invoked when a PHY module is removed or inserted for any OS-specific
1248 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1250 static const char *mod_str[] = {
1251 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1253 struct port_info *pi = &adap->port[port_id];
1254 int mod = pi->phy.modtype;
1256 if (mod != pi->media.ifm_cur->ifm_data)
1257 cxgb_build_medialist(pi);
1259 if (mod == phy_modtype_none)
1260 if_printf(pi->ifp, "PHY module unplugged\n");
1262 KASSERT(mod < ARRAY_SIZE(mod_str),
1263 ("invalid PHY module type %d", mod));
1264 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1269 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1273 * The ifnet might not be allocated before this gets called,
1274 * as this is called early on in attach by t3_prep_adapter
1275 * save the address off in the port structure
1278 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1279 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1283 * Programs the XGMAC based on the settings in the ifnet. These settings
1284 * include MTU, MAC address, mcast addresses, etc.
1287 cxgb_update_mac_settings(struct port_info *p)
1289 struct ifnet *ifp = p->ifp;
1290 struct t3_rx_mode rm;
1291 struct cmac *mac = &p->mac;
1294 PORT_LOCK_ASSERT_OWNED(p);
1296 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1299 if (ifp->if_capenable & IFCAP_VLAN_MTU)
1300 mtu += ETHER_VLAN_ENCAP_LEN;
1302 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1304 t3_mac_set_mtu(mac, mtu);
1305 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1306 t3_mac_set_address(mac, 0, p->hw_addr);
1307 t3_init_rx_mode(&rm, p);
1308 t3_mac_set_rx_mode(mac, &rm);
1313 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1318 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1327 init_tp_parity(struct adapter *adap)
1331 struct cpl_set_tcb_field *greq;
1332 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1334 t3_tp_set_offload_mode(adap, 1);
1336 for (i = 0; i < 16; i++) {
1337 struct cpl_smt_write_req *req;
1339 m = m_gethdr(M_WAITOK, MT_DATA);
1340 req = mtod(m, struct cpl_smt_write_req *);
1341 m->m_len = m->m_pkthdr.len = sizeof(*req);
1342 memset(req, 0, sizeof(*req));
1343 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1344 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1346 t3_mgmt_tx(adap, m);
1349 for (i = 0; i < 2048; i++) {
1350 struct cpl_l2t_write_req *req;
1352 m = m_gethdr(M_WAITOK, MT_DATA);
1353 req = mtod(m, struct cpl_l2t_write_req *);
1354 m->m_len = m->m_pkthdr.len = sizeof(*req);
1355 memset(req, 0, sizeof(*req));
1356 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1357 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1358 req->params = htonl(V_L2T_W_IDX(i));
1359 t3_mgmt_tx(adap, m);
1362 for (i = 0; i < 2048; i++) {
1363 struct cpl_rte_write_req *req;
1365 m = m_gethdr(M_WAITOK, MT_DATA);
1366 req = mtod(m, struct cpl_rte_write_req *);
1367 m->m_len = m->m_pkthdr.len = sizeof(*req);
1368 memset(req, 0, sizeof(*req));
1369 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1370 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1371 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1372 t3_mgmt_tx(adap, m);
1375 m = m_gethdr(M_WAITOK, MT_DATA);
1376 greq = mtod(m, struct cpl_set_tcb_field *);
1377 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1378 memset(greq, 0, sizeof(*greq));
1379 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1380 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1381 greq->mask = htobe64(1);
1382 t3_mgmt_tx(adap, m);
1384 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1385 t3_tp_set_offload_mode(adap, 0);
1390 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1391 * @adap: the adapter
1393 * Sets up RSS to distribute packets to multiple receive queues. We
1394 * configure the RSS CPU lookup table to distribute to the number of HW
1395 * receive queues, and the response queue lookup table to narrow that
1396 * down to the response queues actually configured for each port.
1397 * We always configure the RSS mapping for two ports since the mapping
1398 * table has plenty of entries.
1401 setup_rss(adapter_t *adap)
1405 uint8_t cpus[SGE_QSETS + 1];
1406 uint16_t rspq_map[RSS_TABLE_SIZE];
1408 for (i = 0; i < SGE_QSETS; ++i)
1410 cpus[SGE_QSETS] = 0xff;
1413 for_each_port(adap, i) {
1414 const struct port_info *pi = adap2pinfo(adap, i);
1416 nq[pi->tx_chan] += pi->nqsets;
1418 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1419 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1420 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1423 /* Calculate the reverse RSS map table */
1424 for (i = 0; i < SGE_QSETS; ++i)
1425 adap->rrss_map[i] = 0xff;
1426 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1427 if (adap->rrss_map[rspq_map[i]] == 0xff)
1428 adap->rrss_map[rspq_map[i]] = i;
1430 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1431 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1432 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1437 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1441 struct mngt_pktsched_wr *req;
1443 m = m_gethdr(M_NOWAIT, MT_DATA);
1445 req = mtod(m, struct mngt_pktsched_wr *);
1446 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1447 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1452 req->binding = port;
1453 m->m_len = m->m_pkthdr.len = sizeof(*req);
1454 t3_mgmt_tx(adap, m);
1459 bind_qsets(adapter_t *sc)
1463 for (i = 0; i < (sc)->params.nports; ++i) {
1464 const struct port_info *pi = adap2pinfo(sc, i);
1466 for (j = 0; j < pi->nqsets; ++j) {
1467 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1475 update_tpeeprom(struct adapter *adap)
1477 const struct firmware *tpeeprom;
1480 unsigned int major, minor;
1484 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1486 major = G_TP_VERSION_MAJOR(version);
1487 minor = G_TP_VERSION_MINOR(version);
1488 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1491 rev = t3rev2char(adap);
1492 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1494 tpeeprom = firmware_get(name);
1495 if (tpeeprom == NULL) {
1496 device_printf(adap->dev,
1497 "could not load TP EEPROM: unable to load %s\n",
1502 len = tpeeprom->datasize - 4;
1504 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1506 goto release_tpeeprom;
1508 if (len != TP_SRAM_LEN) {
1509 device_printf(adap->dev,
1510 "%s length is wrong len=%d expected=%d\n", name,
1515 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1519 device_printf(adap->dev,
1520 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1521 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1523 device_printf(adap->dev,
1524 "Protocol SRAM image update in EEPROM failed\n");
1527 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1533 update_tpsram(struct adapter *adap)
1535 const struct firmware *tpsram;
1539 rev = t3rev2char(adap);
1540 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1542 update_tpeeprom(adap);
1544 tpsram = firmware_get(name);
1545 if (tpsram == NULL){
1546 device_printf(adap->dev, "could not load TP SRAM\n");
1549 device_printf(adap->dev, "updating TP SRAM\n");
1551 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1553 goto release_tpsram;
1555 ret = t3_set_proto_sram(adap, tpsram->data);
1557 device_printf(adap->dev, "loading protocol SRAM failed\n");
1560 firmware_put(tpsram, FIRMWARE_UNLOAD);
1566 * cxgb_up - enable the adapter
1567 * @adap: adapter being enabled
1569 * Called when the first port is enabled, this function performs the
1570 * actions necessary to make an adapter operational, such as completing
1571 * the initialization of HW modules, and enabling interrupts.
1574 cxgb_up(struct adapter *sc)
1577 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1579 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1580 __func__, sc->open_device_map));
1582 if ((sc->flags & FULL_INIT_DONE) == 0) {
1584 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1586 if ((sc->flags & FW_UPTODATE) == 0)
1587 if ((err = upgrade_fw(sc)))
1590 if ((sc->flags & TPS_UPTODATE) == 0)
1591 if ((err = update_tpsram(sc)))
1594 if (is_offload(sc) && nfilters != 0) {
1595 sc->params.mc5.nservers = 0;
1598 sc->params.mc5.nfilters = mxf;
1600 sc->params.mc5.nfilters = min(nfilters, mxf);
1603 err = t3_init_hw(sc, 0);
1607 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1608 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1610 err = setup_sge_qsets(sc);
1617 t3_add_configured_sysctls(sc);
1618 sc->flags |= FULL_INIT_DONE;
1625 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1626 is_offload(sc) && init_tp_parity(sc) == 0)
1627 sc->flags |= TP_PARITY_INIT;
1629 if (sc->flags & TP_PARITY_INIT) {
1630 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1631 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1634 if (!(sc->flags & QUEUES_BOUND)) {
1636 setup_hw_filters(sc);
1637 sc->flags |= QUEUES_BOUND;
1640 t3_sge_reset_adapter(sc);
1646 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1647 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1648 * during controller_detach, not here.
1651 cxgb_down(struct adapter *sc)
1654 t3_intr_disable(sc);
1658 * if_init for cxgb ports.
1661 cxgb_init(void *arg)
1663 struct port_info *p = arg;
1664 struct adapter *sc = p->adapter;
1667 cxgb_init_locked(p); /* releases adapter lock */
1668 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1672 cxgb_init_locked(struct port_info *p)
1674 struct adapter *sc = p->adapter;
1675 struct ifnet *ifp = p->ifp;
1676 struct cmac *mac = &p->mac;
1677 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1679 ADAPTER_LOCK_ASSERT_OWNED(sc);
1681 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1683 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1692 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1695 * The code that runs during one-time adapter initialization can sleep
1696 * so it's important not to hold any locks across it.
1698 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1706 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1710 if (isset(&sc->open_device_map, p->port_id) &&
1711 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1715 t3_port_intr_enable(sc, p->port_id);
1716 if (!mac->multiport)
1718 cxgb_update_mac_settings(p);
1719 t3_link_start(&p->phy, mac, &p->link_config);
1720 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1721 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1722 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1725 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1726 struct sge_qset *qs = &sc->sge.qs[i];
1727 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1729 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1730 txq->txq_watchdog.c_cpu);
1734 setbit(&sc->open_device_map, p->port_id);
1735 callout_reset(&p->link_check_ch,
1736 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1737 link_check_callout, p);
1742 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1746 wakeup_one(&sc->flags);
1752 cxgb_uninit_locked(struct port_info *p)
1754 struct adapter *sc = p->adapter;
1757 ADAPTER_LOCK_ASSERT_OWNED(sc);
1759 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1760 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1769 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1773 rc = cxgb_uninit_synchronized(p);
1776 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1778 wakeup_one(&sc->flags);
1785 * Called on "ifconfig down", and from port_detach
1788 cxgb_uninit_synchronized(struct port_info *pi)
1790 struct adapter *sc = pi->adapter;
1791 struct ifnet *ifp = pi->ifp;
1794 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1796 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1799 * Clear this port's bit from the open device map, and then drain all
1800 * the tasks that can access/manipulate this port's port_info or ifp.
1801 * We disable this port's interrupts here and so the slow/ext
1802 * interrupt tasks won't be enqueued. The tick task will continue to
1803 * be enqueued every second but the runs after this drain will not see
1804 * this port in the open device map.
1806 * A well behaved task must take open_device_map into account and ignore
1807 * ports that are not open.
1809 clrbit(&sc->open_device_map, pi->port_id);
1810 t3_port_intr_disable(sc, pi->port_id);
1811 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1812 taskqueue_drain(sc->tq, &sc->tick_task);
1814 callout_drain(&pi->link_check_ch);
1815 taskqueue_drain(sc->tq, &pi->link_check_task);
1818 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1820 /* disable pause frames */
1821 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1823 /* Reset RX FIFO HWM */
1824 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1825 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1829 /* Wait for TXFIFO empty */
1830 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1831 F_TXFIFO_EMPTY, 1, 20, 5);
1834 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1836 pi->phy.ops->power_down(&pi->phy, 1);
1840 pi->link_config.link_ok = 0;
1841 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1843 if (sc->open_device_map == 0)
1844 cxgb_down(pi->adapter);
1850 * Mark lro enabled or disabled in all qsets for this port
1853 cxgb_set_lro(struct port_info *p, int enabled)
1856 struct adapter *adp = p->adapter;
1859 for (i = 0; i < p->nqsets; i++) {
1860 q = &adp->sge.qs[p->first_qset + i];
1861 q->lro.enabled = (enabled != 0);
1867 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1869 struct port_info *p = ifp->if_softc;
1870 struct adapter *sc = p->adapter;
1871 struct ifreq *ifr = (struct ifreq *)data;
1872 int flags, error = 0, mtu;
1878 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1886 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1891 cxgb_update_mac_settings(p);
1902 if (ifp->if_flags & IFF_UP) {
1903 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1904 flags = p->if_flags;
1905 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1906 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
1912 cxgb_update_mac_settings(p);
1917 error = cxgb_init_locked(p);
1918 p->if_flags = ifp->if_flags;
1919 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1920 error = cxgb_uninit_locked(p);
1924 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1929 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1933 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1935 cxgb_update_mac_settings(p);
1943 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1947 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1948 if (mask & IFCAP_TXCSUM) {
1949 ifp->if_capenable ^= IFCAP_TXCSUM;
1950 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1952 if (IFCAP_TSO4 & ifp->if_capenable &&
1953 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1954 ifp->if_capenable &= ~IFCAP_TSO4;
1956 "tso4 disabled due to -txcsum.\n");
1959 if (mask & IFCAP_TXCSUM_IPV6) {
1960 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1961 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1963 if (IFCAP_TSO6 & ifp->if_capenable &&
1964 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1965 ifp->if_capenable &= ~IFCAP_TSO6;
1967 "tso6 disabled due to -txcsum6.\n");
1970 if (mask & IFCAP_RXCSUM)
1971 ifp->if_capenable ^= IFCAP_RXCSUM;
1972 if (mask & IFCAP_RXCSUM_IPV6)
1973 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1976 * Note that we leave CSUM_TSO alone (it is always set). The
1977 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1978 * sending a TSO request our way, so it's sufficient to toggle
1981 if (mask & IFCAP_TSO4) {
1982 if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1983 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1984 if_printf(ifp, "enable txcsum first.\n");
1988 ifp->if_capenable ^= IFCAP_TSO4;
1990 if (mask & IFCAP_TSO6) {
1991 if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1992 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1993 if_printf(ifp, "enable txcsum6 first.\n");
1997 ifp->if_capenable ^= IFCAP_TSO6;
1999 if (mask & IFCAP_LRO) {
2000 ifp->if_capenable ^= IFCAP_LRO;
2002 /* Safe to do this even if cxgb_up not called yet */
2003 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2006 if (mask & IFCAP_TOE4) {
2007 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
2009 error = toe_capability(p, enable);
2011 ifp->if_capenable ^= mask;
2014 if (mask & IFCAP_VLAN_HWTAGGING) {
2015 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2016 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2018 cxgb_update_mac_settings(p);
2022 if (mask & IFCAP_VLAN_MTU) {
2023 ifp->if_capenable ^= IFCAP_VLAN_MTU;
2024 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2026 cxgb_update_mac_settings(p);
2030 if (mask & IFCAP_VLAN_HWTSO)
2031 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2032 if (mask & IFCAP_VLAN_HWCSUM)
2033 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2035 #ifdef VLAN_CAPABILITIES
2036 VLAN_CAPABILITIES(ifp);
2042 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2045 error = ether_ioctl(ifp, command, data);
2052 cxgb_media_change(struct ifnet *ifp)
2054 return (EOPNOTSUPP);
2058 * Translates phy->modtype to the correct Ethernet media subtype.
2061 cxgb_ifm_type(int mod)
2064 case phy_modtype_sr:
2065 return (IFM_10G_SR);
2066 case phy_modtype_lr:
2067 return (IFM_10G_LR);
2068 case phy_modtype_lrm:
2069 return (IFM_10G_LRM);
2070 case phy_modtype_twinax:
2071 return (IFM_10G_TWINAX);
2072 case phy_modtype_twinax_long:
2073 return (IFM_10G_TWINAX_LONG);
2074 case phy_modtype_none:
2076 case phy_modtype_unknown:
2077 return (IFM_UNKNOWN);
2080 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2081 return (IFM_UNKNOWN);
2085 * Rebuilds the ifmedia list for this port, and sets the current media.
2088 cxgb_build_medialist(struct port_info *p)
2090 struct cphy *phy = &p->phy;
2091 struct ifmedia *media = &p->media;
2092 int mod = phy->modtype;
2093 int m = IFM_ETHER | IFM_FDX;
2097 ifmedia_removeall(media);
2098 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2101 if (phy->caps & SUPPORTED_10000baseT_Full)
2102 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2104 if (phy->caps & SUPPORTED_1000baseT_Full)
2105 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2107 if (phy->caps & SUPPORTED_100baseT_Full)
2108 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2110 if (phy->caps & SUPPORTED_10baseT_Full)
2111 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2113 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2114 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2116 } else if (phy->caps & SUPPORTED_TP) {
2119 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2120 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2122 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2123 ifmedia_set(media, m | IFM_10G_CX4);
2125 } else if (phy->caps & SUPPORTED_FIBRE &&
2126 phy->caps & SUPPORTED_10000baseT_Full) {
2127 /* 10G optical (but includes SFP+ twinax) */
2129 m |= cxgb_ifm_type(mod);
2130 if (IFM_SUBTYPE(m) == IFM_NONE)
2133 ifmedia_add(media, m, mod, NULL);
2134 ifmedia_set(media, m);
2136 } else if (phy->caps & SUPPORTED_FIBRE &&
2137 phy->caps & SUPPORTED_1000baseT_Full) {
2140 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2141 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2142 ifmedia_set(media, m | IFM_1000_SX);
2145 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2153 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2155 struct port_info *p = ifp->if_softc;
2156 struct ifmedia_entry *cur = p->media.ifm_cur;
2157 int speed = p->link_config.speed;
2159 if (cur->ifm_data != p->phy.modtype) {
2160 cxgb_build_medialist(p);
2161 cur = p->media.ifm_cur;
2164 ifmr->ifm_status = IFM_AVALID;
2165 if (!p->link_config.link_ok)
2168 ifmr->ifm_status |= IFM_ACTIVE;
2171 * active and current will differ iff current media is autoselect. That
2172 * can happen only for copper RJ45.
2174 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2176 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2177 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2179 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2180 if (speed == SPEED_10000)
2181 ifmr->ifm_active |= IFM_10G_T;
2182 else if (speed == SPEED_1000)
2183 ifmr->ifm_active |= IFM_1000_T;
2184 else if (speed == SPEED_100)
2185 ifmr->ifm_active |= IFM_100_TX;
2186 else if (speed == SPEED_10)
2187 ifmr->ifm_active |= IFM_10_T;
2189 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2194 cxgb_get_counter(struct ifnet *ifp, ift_counter c)
2196 struct port_info *pi = ifp->if_softc;
2197 struct adapter *sc = pi->adapter;
2198 struct cmac *mac = &pi->mac;
2199 struct mac_stats *mstats = &mac->stats;
2201 cxgb_refresh_stats(pi);
2204 case IFCOUNTER_IPACKETS:
2205 return (mstats->rx_frames);
2207 case IFCOUNTER_IERRORS:
2208 return (mstats->rx_jabber + mstats->rx_data_errs +
2209 mstats->rx_sequence_errs + mstats->rx_runt +
2210 mstats->rx_too_long + mstats->rx_mac_internal_errs +
2211 mstats->rx_short + mstats->rx_fcs_errs);
2213 case IFCOUNTER_OPACKETS:
2214 return (mstats->tx_frames);
2216 case IFCOUNTER_OERRORS:
2217 return (mstats->tx_excess_collisions + mstats->tx_underrun +
2218 mstats->tx_len_errs + mstats->tx_mac_internal_errs +
2219 mstats->tx_excess_deferral + mstats->tx_fcs_errs);
2221 case IFCOUNTER_COLLISIONS:
2222 return (mstats->tx_total_collisions);
2224 case IFCOUNTER_IBYTES:
2225 return (mstats->rx_octets);
2227 case IFCOUNTER_OBYTES:
2228 return (mstats->tx_octets);
2230 case IFCOUNTER_IMCASTS:
2231 return (mstats->rx_mcast_frames);
2233 case IFCOUNTER_OMCASTS:
2234 return (mstats->tx_mcast_frames);
2236 case IFCOUNTER_IQDROPS:
2237 return (mstats->rx_cong_drops);
2239 case IFCOUNTER_OQDROPS: {
2244 if (sc->flags & FULL_INIT_DONE) {
2245 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
2246 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
2254 return (if_get_counter_default(ifp, c));
2259 cxgb_async_intr(void *data)
2261 adapter_t *sc = data;
2263 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2264 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2265 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2269 link_check_callout(void *arg)
2271 struct port_info *pi = arg;
2272 struct adapter *sc = pi->adapter;
2274 if (!isset(&sc->open_device_map, pi->port_id))
2277 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2281 check_link_status(void *arg, int pending)
2283 struct port_info *pi = arg;
2284 struct adapter *sc = pi->adapter;
2286 if (!isset(&sc->open_device_map, pi->port_id))
2289 t3_link_changed(sc, pi->port_id);
2291 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
2292 pi->link_config.link_ok == 0)
2293 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2297 t3_os_link_intr(struct port_info *pi)
2300 * Schedule a link check in the near future. If the link is flapping
2301 * rapidly we'll keep resetting the callout and delaying the check until
2302 * things stabilize a bit.
2304 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2308 check_t3b2_mac(struct adapter *sc)
2312 if (sc->flags & CXGB_SHUTDOWN)
2315 for_each_port(sc, i) {
2316 struct port_info *p = &sc->port[i];
2319 struct ifnet *ifp = p->ifp;
2322 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2323 !p->link_config.link_ok)
2326 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2327 ("%s: state mismatch (drv_flags %x, device_map %x)",
2328 __func__, ifp->if_drv_flags, sc->open_device_map));
2331 status = t3b2_mac_watchdog_task(&p->mac);
2333 p->mac.stats.num_toggled++;
2334 else if (status == 2) {
2335 struct cmac *mac = &p->mac;
2337 cxgb_update_mac_settings(p);
2338 t3_link_start(&p->phy, mac, &p->link_config);
2339 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2340 t3_port_intr_enable(sc, p->port_id);
2341 p->mac.stats.num_resets++;
2348 cxgb_tick(void *arg)
2350 adapter_t *sc = (adapter_t *)arg;
2352 if (sc->flags & CXGB_SHUTDOWN)
2355 taskqueue_enqueue(sc->tq, &sc->tick_task);
2356 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2360 cxgb_refresh_stats(struct port_info *pi)
2363 const struct timeval interval = {0, 250000}; /* 250ms */
2366 timevalsub(&tv, &interval);
2367 if (timevalcmp(&tv, &pi->last_refreshed, <))
2371 t3_mac_update_stats(&pi->mac);
2373 getmicrotime(&pi->last_refreshed);
2377 cxgb_tick_handler(void *arg, int count)
2379 adapter_t *sc = (adapter_t *)arg;
2380 const struct adapter_params *p = &sc->params;
2382 uint32_t cause, reset;
2384 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2387 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2390 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2392 struct sge_qset *qs = &sc->sge.qs[0];
2395 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2398 for (i = 0; i < SGE_QSETS; i++) {
2400 qs[i].rspq.starved++;
2404 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2406 for (i = 0; i < SGE_QSETS * 2; i++) {
2408 qs[i / 2].fl[i % 2].empty++;
2414 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2415 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2418 for (i = 0; i < sc->params.nports; i++) {
2419 struct port_info *pi = &sc->port[i];
2420 struct cmac *mac = &pi->mac;
2422 if (!isset(&sc->open_device_map, pi->port_id))
2425 cxgb_refresh_stats(pi);
2430 /* Count rx fifo overflows, once per second */
2431 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2433 if (cause & F_RXFIFO_OVERFLOW) {
2434 mac->stats.rx_fifo_ovfl++;
2435 reset |= F_RXFIFO_OVERFLOW;
2437 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2442 touch_bars(device_t dev)
2447 #if !defined(__LP64__) && 0
2450 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2451 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2452 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2453 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2454 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2455 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2460 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2464 u32 aligned_offset, aligned_len, *p;
2465 struct adapter *adapter = pi->adapter;
2468 aligned_offset = offset & ~3;
2469 aligned_len = (len + (offset & 3) + 3) & ~3;
2471 if (aligned_offset != offset || aligned_len != len) {
2472 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2475 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2476 if (!err && aligned_len > 4)
2477 err = t3_seeprom_read(adapter,
2478 aligned_offset + aligned_len - 4,
2479 (u32 *)&buf[aligned_len - 4]);
2482 memcpy(buf + (offset & 3), data, len);
2484 buf = (uint8_t *)(uintptr_t)data;
2486 err = t3_seeprom_wp(adapter, 0);
2490 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2491 err = t3_seeprom_write(adapter, aligned_offset, *p);
2492 aligned_offset += 4;
2496 err = t3_seeprom_wp(adapter, 1);
2499 free(buf, M_DEVBUF);
2505 in_range(int val, int lo, int hi)
2507 return val < 0 || (val <= hi && val >= lo);
2511 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2517 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2523 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2524 int fflag, struct thread *td)
2527 struct port_info *pi = dev->si_drv1;
2528 adapter_t *sc = pi->adapter;
2530 #ifdef PRIV_SUPPORTED
2531 if (priv_check(td, PRIV_DRIVER)) {
2533 printf("user does not have access to privileged ioctls\n");
2539 printf("user does not have access to privileged ioctls\n");
2545 case CHELSIO_GET_MIIREG: {
2547 struct cphy *phy = &pi->phy;
2548 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2550 if (!phy->mdio_read)
2551 return (EOPNOTSUPP);
2553 mmd = mid->phy_id >> 8;
2556 else if (mmd > MDIO_DEV_VEND2)
2559 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2560 mid->reg_num, &val);
2562 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2563 mid->reg_num & 0x1f, &val);
2568 case CHELSIO_SET_MIIREG: {
2569 struct cphy *phy = &pi->phy;
2570 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2572 if (!phy->mdio_write)
2573 return (EOPNOTSUPP);
2575 mmd = mid->phy_id >> 8;
2578 else if (mmd > MDIO_DEV_VEND2)
2581 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2582 mmd, mid->reg_num, mid->val_in);
2584 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2585 mid->reg_num & 0x1f,
2589 case CHELSIO_SETREG: {
2590 struct ch_reg *edata = (struct ch_reg *)data;
2591 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2593 t3_write_reg(sc, edata->addr, edata->val);
2596 case CHELSIO_GETREG: {
2597 struct ch_reg *edata = (struct ch_reg *)data;
2598 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2600 edata->val = t3_read_reg(sc, edata->addr);
2603 case CHELSIO_GET_SGE_CONTEXT: {
2604 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2605 mtx_lock_spin(&sc->sge.reg_lock);
2606 switch (ecntxt->cntxt_type) {
2607 case CNTXT_TYPE_EGRESS:
2608 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2612 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2615 case CNTXT_TYPE_RSP:
2616 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2620 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2627 mtx_unlock_spin(&sc->sge.reg_lock);
2630 case CHELSIO_GET_SGE_DESC: {
2631 struct ch_desc *edesc = (struct ch_desc *)data;
2633 if (edesc->queue_num >= SGE_QSETS * 6)
2635 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2636 edesc->queue_num % 6, edesc->idx, edesc->data);
2642 case CHELSIO_GET_QSET_PARAMS: {
2643 struct qset_params *q;
2644 struct ch_qset_params *t = (struct ch_qset_params *)data;
2645 int q1 = pi->first_qset;
2646 int nqsets = pi->nqsets;
2649 if (t->qset_idx >= nqsets)
2652 i = q1 + t->qset_idx;
2653 q = &sc->params.sge.qset[i];
2654 t->rspq_size = q->rspq_size;
2655 t->txq_size[0] = q->txq_size[0];
2656 t->txq_size[1] = q->txq_size[1];
2657 t->txq_size[2] = q->txq_size[2];
2658 t->fl_size[0] = q->fl_size;
2659 t->fl_size[1] = q->jumbo_size;
2660 t->polling = q->polling;
2662 t->intr_lat = q->coalesce_usecs;
2663 t->cong_thres = q->cong_thres;
2666 if ((sc->flags & FULL_INIT_DONE) == 0)
2668 else if (sc->flags & USING_MSIX)
2669 t->vector = rman_get_start(sc->msix_irq_res[i]);
2671 t->vector = rman_get_start(sc->irq_res);
2675 case CHELSIO_GET_QSET_NUM: {
2676 struct ch_reg *edata = (struct ch_reg *)data;
2677 edata->val = pi->nqsets;
2680 case CHELSIO_LOAD_FW: {
2683 struct ch_mem_range *t = (struct ch_mem_range *)data;
2686 * You're allowed to load a firmware only before FULL_INIT_DONE
2688 * FW_UPTODATE is also set so the rest of the initialization
2689 * will not overwrite what was loaded here. This gives you the
2690 * flexibility to load any firmware (and maybe shoot yourself in
2695 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2700 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2704 error = copyin(t->buf, fw_data, t->len);
2707 error = -t3_load_fw(sc, fw_data, t->len);
2709 if (t3_get_fw_version(sc, &vers) == 0) {
2710 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2711 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2712 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2716 sc->flags |= FW_UPTODATE;
2718 free(fw_data, M_DEVBUF);
2722 case CHELSIO_LOAD_BOOT: {
2724 struct ch_mem_range *t = (struct ch_mem_range *)data;
2726 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2730 error = copyin(t->buf, boot_data, t->len);
2732 error = -t3_load_boot(sc, boot_data, t->len);
2734 free(boot_data, M_DEVBUF);
2737 case CHELSIO_GET_PM: {
2738 struct ch_pm *m = (struct ch_pm *)data;
2739 struct tp_params *p = &sc->params.tp;
2741 if (!is_offload(sc))
2742 return (EOPNOTSUPP);
2744 m->tx_pg_sz = p->tx_pg_size;
2745 m->tx_num_pg = p->tx_num_pgs;
2746 m->rx_pg_sz = p->rx_pg_size;
2747 m->rx_num_pg = p->rx_num_pgs;
2748 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2752 case CHELSIO_SET_PM: {
2753 struct ch_pm *m = (struct ch_pm *)data;
2754 struct tp_params *p = &sc->params.tp;
2756 if (!is_offload(sc))
2757 return (EOPNOTSUPP);
2758 if (sc->flags & FULL_INIT_DONE)
2761 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2762 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2763 return (EINVAL); /* not power of 2 */
2764 if (!(m->rx_pg_sz & 0x14000))
2765 return (EINVAL); /* not 16KB or 64KB */
2766 if (!(m->tx_pg_sz & 0x1554000))
2768 if (m->tx_num_pg == -1)
2769 m->tx_num_pg = p->tx_num_pgs;
2770 if (m->rx_num_pg == -1)
2771 m->rx_num_pg = p->rx_num_pgs;
2772 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2774 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2775 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2778 p->rx_pg_size = m->rx_pg_sz;
2779 p->tx_pg_size = m->tx_pg_sz;
2780 p->rx_num_pgs = m->rx_num_pg;
2781 p->tx_num_pgs = m->tx_num_pg;
2784 case CHELSIO_SETMTUTAB: {
2785 struct ch_mtus *m = (struct ch_mtus *)data;
2788 if (!is_offload(sc))
2789 return (EOPNOTSUPP);
2790 if (offload_running(sc))
2792 if (m->nmtus != NMTUS)
2794 if (m->mtus[0] < 81) /* accommodate SACK */
2798 * MTUs must be in ascending order
2800 for (i = 1; i < NMTUS; ++i)
2801 if (m->mtus[i] < m->mtus[i - 1])
2804 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2807 case CHELSIO_GETMTUTAB: {
2808 struct ch_mtus *m = (struct ch_mtus *)data;
2810 if (!is_offload(sc))
2811 return (EOPNOTSUPP);
2813 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2817 case CHELSIO_GET_MEM: {
2818 struct ch_mem_range *t = (struct ch_mem_range *)data;
2824 * Use these to avoid modifying len/addr in the return
2827 uint32_t len = t->len, addr = t->addr;
2829 if (!is_offload(sc))
2830 return (EOPNOTSUPP);
2831 if (!(sc->flags & FULL_INIT_DONE))
2832 return (EIO); /* need the memory controllers */
2833 if ((addr & 0x7) || (len & 0x7))
2835 if (t->mem_id == MEM_CM)
2837 else if (t->mem_id == MEM_PMRX)
2839 else if (t->mem_id == MEM_PMTX)
2846 * bits 0..9: chip version
2847 * bits 10..15: chip revision
2849 t->version = 3 | (sc->params.rev << 10);
2852 * Read 256 bytes at a time as len can be large and we don't
2853 * want to use huge intermediate buffers.
2855 useraddr = (uint8_t *)t->buf;
2857 unsigned int chunk = min(len, sizeof(buf));
2859 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2862 if (copyout(buf, useraddr, chunk))
2870 case CHELSIO_READ_TCAM_WORD: {
2871 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2873 if (!is_offload(sc))
2874 return (EOPNOTSUPP);
2875 if (!(sc->flags & FULL_INIT_DONE))
2876 return (EIO); /* need MC5 */
2877 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2880 case CHELSIO_SET_TRACE_FILTER: {
2881 struct ch_trace *t = (struct ch_trace *)data;
2882 const struct trace_params *tp;
2884 tp = (const struct trace_params *)&t->sip;
2886 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2889 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2893 case CHELSIO_SET_PKTSCHED: {
2894 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2895 if (sc->open_device_map == 0)
2897 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2901 case CHELSIO_IFCONF_GETREGS: {
2902 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2903 int reglen = cxgb_get_regs_len();
2904 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2908 if (regs->len > reglen)
2910 else if (regs->len < reglen)
2914 cxgb_get_regs(sc, regs, buf);
2915 error = copyout(buf, regs->data, reglen);
2917 free(buf, M_DEVBUF);
2921 case CHELSIO_SET_HW_SCHED: {
2922 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2923 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2925 if ((sc->flags & FULL_INIT_DONE) == 0)
2926 return (EAGAIN); /* need TP to be initialized */
2927 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2928 !in_range(t->channel, 0, 1) ||
2929 !in_range(t->kbps, 0, 10000000) ||
2930 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2931 !in_range(t->flow_ipg, 0,
2932 dack_ticks_to_usec(sc, 0x7ff)))
2936 error = t3_config_sched(sc, t->kbps, t->sched);
2940 if (t->class_ipg >= 0)
2941 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2942 if (t->flow_ipg >= 0) {
2943 t->flow_ipg *= 1000; /* us -> ns */
2944 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2947 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2949 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2950 bit, t->mode ? bit : 0);
2952 if (t->channel >= 0)
2953 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2954 1 << t->sched, t->channel << t->sched);
2957 case CHELSIO_GET_EEPROM: {
2959 struct ch_eeprom *e = (struct ch_eeprom *)data;
2960 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2965 e->magic = EEPROM_MAGIC;
2966 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2967 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2970 error = copyout(buf + e->offset, e->data, e->len);
2972 free(buf, M_DEVBUF);
2975 case CHELSIO_CLEAR_STATS: {
2976 if (!(sc->flags & FULL_INIT_DONE))
2980 t3_mac_update_stats(&pi->mac);
2981 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
2985 case CHELSIO_GET_UP_LA: {
2986 struct ch_up_la *la = (struct ch_up_la *)data;
2987 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
2991 if (la->bufsize < LA_BUFSIZE)
2995 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
2998 error = copyout(buf, la->data, la->bufsize);
3000 free(buf, M_DEVBUF);
3003 case CHELSIO_GET_UP_IOQS: {
3004 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3005 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3011 if (ioqs->bufsize < IOQS_BUFSIZE)
3015 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3018 v = (uint32_t *)buf;
3020 ioqs->ioq_rx_enable = *v++;
3021 ioqs->ioq_tx_enable = *v++;
3022 ioqs->ioq_rx_status = *v++;
3023 ioqs->ioq_tx_status = *v++;
3025 error = copyout(v, ioqs->data, ioqs->bufsize);
3028 free(buf, M_DEVBUF);
3031 case CHELSIO_SET_FILTER: {
3032 struct ch_filter *f = (struct ch_filter *)data;
3033 struct filter_info *p;
3034 unsigned int nfilters = sc->params.mc5.nfilters;
3036 if (!is_offload(sc))
3037 return (EOPNOTSUPP); /* No TCAM */
3038 if (!(sc->flags & FULL_INIT_DONE))
3039 return (EAGAIN); /* mc5 not setup yet */
3041 return (EBUSY); /* TOE will use TCAM */
3044 if (f->filter_id >= nfilters ||
3045 (f->val.dip && f->mask.dip != 0xffffffff) ||
3046 (f->val.sport && f->mask.sport != 0xffff) ||
3047 (f->val.dport && f->mask.dport != 0xffff) ||
3048 (f->val.vlan && f->mask.vlan != 0xfff) ||
3049 (f->val.vlan_prio &&
3050 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3051 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3052 f->qset >= SGE_QSETS ||
3053 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3056 /* Was allocated with M_WAITOK */
3057 KASSERT(sc->filters, ("filter table NULL\n"));
3059 p = &sc->filters[f->filter_id];
3063 bzero(p, sizeof(*p));
3064 p->sip = f->val.sip;
3065 p->sip_mask = f->mask.sip;
3066 p->dip = f->val.dip;
3067 p->sport = f->val.sport;
3068 p->dport = f->val.dport;
3069 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3070 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3072 p->mac_hit = f->mac_hit;
3073 p->mac_vld = f->mac_addr_idx != 0xffff;
3074 p->mac_idx = f->mac_addr_idx;
3075 p->pkt_type = f->proto;
3076 p->report_filter_id = f->want_filter_id;
3081 error = set_filter(sc, f->filter_id, p);
3086 case CHELSIO_DEL_FILTER: {
3087 struct ch_filter *f = (struct ch_filter *)data;
3088 struct filter_info *p;
3089 unsigned int nfilters = sc->params.mc5.nfilters;
3091 if (!is_offload(sc))
3092 return (EOPNOTSUPP);
3093 if (!(sc->flags & FULL_INIT_DONE))
3095 if (nfilters == 0 || sc->filters == NULL)
3097 if (f->filter_id >= nfilters)
3100 p = &sc->filters[f->filter_id];
3104 return (EFAULT); /* Read "Bad address" as "Bad index" */
3106 bzero(p, sizeof(*p));
3107 p->sip = p->sip_mask = 0xffffffff;
3109 p->vlan_prio = FILTER_NO_VLAN_PRI;
3111 error = set_filter(sc, f->filter_id, p);
3114 case CHELSIO_GET_FILTER: {
3115 struct ch_filter *f = (struct ch_filter *)data;
3116 struct filter_info *p;
3117 unsigned int i, nfilters = sc->params.mc5.nfilters;
3119 if (!is_offload(sc))
3120 return (EOPNOTSUPP);
3121 if (!(sc->flags & FULL_INIT_DONE))
3123 if (nfilters == 0 || sc->filters == NULL)
3126 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3127 for (; i < nfilters; i++) {
3128 p = &sc->filters[i];
3132 bzero(f, sizeof(*f));
3135 f->val.sip = p->sip;
3136 f->mask.sip = p->sip_mask;
3137 f->val.dip = p->dip;
3138 f->mask.dip = p->dip ? 0xffffffff : 0;
3139 f->val.sport = p->sport;
3140 f->mask.sport = p->sport ? 0xffff : 0;
3141 f->val.dport = p->dport;
3142 f->mask.dport = p->dport ? 0xffff : 0;
3143 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3144 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3145 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3147 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3148 0 : FILTER_NO_VLAN_PRI;
3149 f->mac_hit = p->mac_hit;
3150 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3151 f->proto = p->pkt_type;
3152 f->want_filter_id = p->report_filter_id;
3161 f->filter_id = 0xffffffff;
3165 return (EOPNOTSUPP);
3172 static __inline void
3173 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3176 uint32_t *p = (uint32_t *)(buf + start);
3178 for ( ; start <= end; start += sizeof(uint32_t))
3179 *p++ = t3_read_reg(ap, start);
3182 #define T3_REGMAP_SIZE (3 * 1024)
3184 cxgb_get_regs_len(void)
3186 return T3_REGMAP_SIZE;
3190 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3195 * bits 0..9: chip version
3196 * bits 10..15: chip revision
3197 * bit 31: set for PCIe cards
3199 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3202 * We skip the MAC statistics registers because they are clear-on-read.
3203 * Also reading multi-register stats would need to synchronize with the
3204 * periodic mac stats accumulation. Hard to justify the complexity.
3206 memset(buf, 0, cxgb_get_regs_len());
3207 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3208 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3209 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3210 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3211 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3212 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3213 XGM_REG(A_XGM_SERDES_STAT3, 1));
3214 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3215 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3219 alloc_filters(struct adapter *sc)
3221 struct filter_info *p;
3222 unsigned int nfilters = sc->params.mc5.nfilters;
3227 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3230 p = &sc->filters[nfilters - 1];
3232 p->vlan_prio = FILTER_NO_VLAN_PRI;
3233 p->pass = p->rss = p->valid = p->locked = 1;
3239 setup_hw_filters(struct adapter *sc)
3242 unsigned int nfilters = sc->params.mc5.nfilters;
3247 t3_enable_filters(sc);
3249 for (i = rc = 0; i < nfilters && !rc; i++) {
3250 if (sc->filters[i].locked)
3251 rc = set_filter(sc, i, &sc->filters[i]);
3258 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3262 struct ulp_txpkt *txpkt;
3263 struct work_request_hdr *wr;
3264 struct cpl_pass_open_req *oreq;
3265 struct cpl_set_tcb_field *sreq;
3267 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3268 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3270 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3271 sc->params.mc5.nfilters;
3273 m = m_gethdr(M_WAITOK, MT_DATA);
3274 m->m_len = m->m_pkthdr.len = len;
3275 bzero(mtod(m, char *), len);
3277 wr = mtod(m, struct work_request_hdr *);
3278 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3280 oreq = (struct cpl_pass_open_req *)(wr + 1);
3281 txpkt = (struct ulp_txpkt *)oreq;
3282 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3283 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3284 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3285 oreq->local_port = htons(f->dport);
3286 oreq->peer_port = htons(f->sport);
3287 oreq->local_ip = htonl(f->dip);
3288 oreq->peer_ip = htonl(f->sip);
3289 oreq->peer_netmask = htonl(f->sip_mask);
3291 oreq->opt0l = htonl(F_NO_OFFLOAD);
3292 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3293 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3294 V_VLAN_PRI(f->vlan_prio >> 1) |
3295 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3296 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3297 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3299 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3300 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3301 (f->report_filter_id << 15) | (1 << 23) |
3302 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3303 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3306 if (f->pass && !f->rss) {
3307 len = sizeof(*sreq);
3308 m = m_gethdr(M_WAITOK, MT_DATA);
3309 m->m_len = m->m_pkthdr.len = len;
3310 bzero(mtod(m, char *), len);
3311 sreq = mtod(m, struct cpl_set_tcb_field *);
3312 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3313 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3314 (u64)sc->rrss_map[f->qset] << 19);
3321 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3322 unsigned int word, u64 mask, u64 val)
3324 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3325 req->reply = V_NO_REPLY(1);
3327 req->word = htons(word);
3328 req->mask = htobe64(mask);
3329 req->val = htobe64(val);
3333 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3334 unsigned int word, u64 mask, u64 val)
3336 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3338 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3339 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3340 mk_set_tcb_field(req, tid, word, mask, val);
3344 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3348 mtx_lock(&t3_list_lock);
3349 SLIST_FOREACH(sc, &t3_list, link) {
3351 * func should not make any assumptions about what state sc is
3352 * in - the only guarantee is that sc->sc_lock is a valid lock.
3356 mtx_unlock(&t3_list_lock);
3361 toe_capability(struct port_info *pi, int enable)
3364 struct adapter *sc = pi->adapter;
3366 ADAPTER_LOCK_ASSERT_OWNED(sc);
3368 if (!is_offload(sc))
3372 if (!(sc->flags & FULL_INIT_DONE)) {
3374 "You must enable a cxgb interface first\n");
3378 if (isset(&sc->offload_map, pi->port_id))
3381 if (!(sc->flags & TOM_INIT_DONE)) {
3382 rc = t3_activate_uld(sc, ULD_TOM);
3385 "You must kldload t3_tom.ko before trying "
3386 "to enable TOE on a cxgb interface.\n");
3390 KASSERT(sc->tom_softc != NULL,
3391 ("%s: TOM activated but softc NULL", __func__));
3392 KASSERT(sc->flags & TOM_INIT_DONE,
3393 ("%s: TOM activated but flag not set", __func__));
3396 setbit(&sc->offload_map, pi->port_id);
3399 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3400 * enabled on any port. Need to figure out how to enable,
3401 * disable, load, and unload iWARP cleanly.
3403 if (!isset(&sc->offload_map, MAX_NPORTS) &&
3404 t3_activate_uld(sc, ULD_IWARP) == 0)
3405 setbit(&sc->offload_map, MAX_NPORTS);
3407 if (!isset(&sc->offload_map, pi->port_id))
3410 KASSERT(sc->flags & TOM_INIT_DONE,
3411 ("%s: TOM never initialized?", __func__));
3412 clrbit(&sc->offload_map, pi->port_id);
3419 * Add an upper layer driver to the global list.
3422 t3_register_uld(struct uld_info *ui)
3427 mtx_lock(&t3_uld_list_lock);
3428 SLIST_FOREACH(u, &t3_uld_list, link) {
3429 if (u->uld_id == ui->uld_id) {
3435 SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3438 mtx_unlock(&t3_uld_list_lock);
3443 t3_unregister_uld(struct uld_info *ui)
3448 mtx_lock(&t3_uld_list_lock);
3450 SLIST_FOREACH(u, &t3_uld_list, link) {
3452 if (ui->refcount > 0) {
3457 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3463 mtx_unlock(&t3_uld_list_lock);
3468 t3_activate_uld(struct adapter *sc, int id)
3471 struct uld_info *ui;
3473 mtx_lock(&t3_uld_list_lock);
3475 SLIST_FOREACH(ui, &t3_uld_list, link) {
3476 if (ui->uld_id == id) {
3477 rc = ui->activate(sc);
3484 mtx_unlock(&t3_uld_list_lock);
3490 t3_deactivate_uld(struct adapter *sc, int id)
3493 struct uld_info *ui;
3495 mtx_lock(&t3_uld_list_lock);
3497 SLIST_FOREACH(ui, &t3_uld_list, link) {
3498 if (ui->uld_id == id) {
3499 rc = ui->deactivate(sc);
3506 mtx_unlock(&t3_uld_list_lock);
3512 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3520 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3522 uintptr_t *loc, new;
3524 if (opcode >= NUM_CPL_HANDLERS)
3527 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3528 loc = (uintptr_t *) &sc->cpl_handler[opcode];
3529 atomic_store_rel_ptr(loc, new);
3536 cxgbc_mod_event(module_t mod, int cmd, void *arg)
3542 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3543 SLIST_INIT(&t3_list);
3545 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3546 SLIST_INIT(&t3_uld_list);
3552 mtx_lock(&t3_uld_list_lock);
3553 if (!SLIST_EMPTY(&t3_uld_list)) {
3555 mtx_unlock(&t3_uld_list_lock);
3558 mtx_unlock(&t3_uld_list_lock);
3559 mtx_destroy(&t3_uld_list_lock);
3561 mtx_lock(&t3_list_lock);
3562 if (!SLIST_EMPTY(&t3_list)) {
3564 mtx_unlock(&t3_list_lock);
3567 mtx_unlock(&t3_list_lock);
3568 mtx_destroy(&t3_list_lock);