1 /**************************************************************************
3 Copyright (c) 2007-2009, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/pciio.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/bus_dma.h>
45 #include <sys/ioccom.h>
47 #include <sys/linker.h>
48 #include <sys/firmware.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/queue.h>
55 #include <sys/taskqueue.h>
59 #include <net/ethernet.h>
61 #include <net/if_arp.h>
62 #include <net/if_dl.h>
63 #include <net/if_media.h>
64 #include <net/if_types.h>
65 #include <net/if_vlan_var.h>
67 #include <netinet/in_systm.h>
68 #include <netinet/in.h>
69 #include <netinet/if_ether.h>
70 #include <netinet/ip.h>
71 #include <netinet/ip.h>
72 #include <netinet/tcp.h>
73 #include <netinet/udp.h>
75 #include <dev/pci/pcireg.h>
76 #include <dev/pci/pcivar.h>
77 #include <dev/pci/pci_private.h>
79 #include <cxgb_include.h>
85 static int cxgb_setup_interrupts(adapter_t *);
86 static void cxgb_teardown_interrupts(adapter_t *);
87 static void cxgb_init(void *);
88 static int cxgb_init_locked(struct port_info *);
89 static int cxgb_uninit_locked(struct port_info *);
90 static int cxgb_uninit_synchronized(struct port_info *);
91 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
92 static int cxgb_media_change(struct ifnet *);
93 static int cxgb_ifm_type(int);
94 static void cxgb_build_medialist(struct port_info *);
95 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
96 static int setup_sge_qsets(adapter_t *);
97 static void cxgb_async_intr(void *);
98 static void cxgb_tick_handler(void *, int);
99 static void cxgb_tick(void *);
100 static void link_check_callout(void *);
101 static void check_link_status(void *, int);
102 static void setup_rss(adapter_t *sc);
103 static int alloc_filters(struct adapter *);
104 static int setup_hw_filters(struct adapter *);
105 static int set_filter(struct adapter *, int, const struct filter_info *);
106 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
107 unsigned int, u64, u64);
108 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
109 unsigned int, u64, u64);
111 /* Attachment glue for the PCI controller end of the device. Each port of
112 * the device is attached separately, as defined later.
114 static int cxgb_controller_probe(device_t);
115 static int cxgb_controller_attach(device_t);
116 static int cxgb_controller_detach(device_t);
117 static void cxgb_free(struct adapter *);
118 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
120 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
121 static int cxgb_get_regs_len(void);
122 static int offload_open(struct port_info *pi);
123 static void touch_bars(device_t dev);
124 static int offload_close(struct t3cdev *tdev);
125 static void cxgb_update_mac_settings(struct port_info *p);
127 static device_method_t cxgb_controller_methods[] = {
128 DEVMETHOD(device_probe, cxgb_controller_probe),
129 DEVMETHOD(device_attach, cxgb_controller_attach),
130 DEVMETHOD(device_detach, cxgb_controller_detach),
135 static driver_t cxgb_controller_driver = {
137 cxgb_controller_methods,
138 sizeof(struct adapter)
141 static devclass_t cxgb_controller_devclass;
142 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
145 * Attachment glue for the ports. Attachment is done directly to the
148 static int cxgb_port_probe(device_t);
149 static int cxgb_port_attach(device_t);
150 static int cxgb_port_detach(device_t);
152 static device_method_t cxgb_port_methods[] = {
153 DEVMETHOD(device_probe, cxgb_port_probe),
154 DEVMETHOD(device_attach, cxgb_port_attach),
155 DEVMETHOD(device_detach, cxgb_port_detach),
159 static driver_t cxgb_port_driver = {
165 static d_ioctl_t cxgb_extension_ioctl;
166 static d_open_t cxgb_extension_open;
167 static d_close_t cxgb_extension_close;
169 static struct cdevsw cxgb_cdevsw = {
170 .d_version = D_VERSION,
172 .d_open = cxgb_extension_open,
173 .d_close = cxgb_extension_close,
174 .d_ioctl = cxgb_extension_ioctl,
178 static devclass_t cxgb_port_devclass;
179 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
182 * The driver uses the best interrupt scheme available on a platform in the
183 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
184 * of these schemes the driver may consider as follows:
186 * msi = 2: choose from among all three options
187 * msi = 1 : only consider MSI and pin interrupts
188 * msi = 0: force pin interrupts
190 static int msi_allowed = 2;
192 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
193 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
194 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
195 "MSI-X, MSI, INTx selector");
198 * The driver enables offload as a default.
199 * To disable it, use ofld_disable = 1.
201 static int ofld_disable = 0;
202 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
203 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
204 "disable ULP offload");
207 * The driver uses an auto-queue algorithm by default.
208 * To disable it and force a single queue-set per port, use multiq = 0
210 static int multiq = 1;
211 TUNABLE_INT("hw.cxgb.multiq", &multiq);
212 SYSCTL_UINT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
213 "use min(ncpus/ports, 8) queue-sets per port");
216 * By default the driver will not update the firmware unless
217 * it was compiled against a newer version
220 static int force_fw_update = 0;
221 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
222 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
223 "update firmware even if up to date");
225 int cxgb_use_16k_clusters = -1;
226 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
227 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
228 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
230 static int nfilters = -1;
231 TUNABLE_INT("hw.cxgb.nfilters", &nfilters);
232 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
233 &nfilters, 0, "max number of entries in the filter table");
236 MAX_TXQ_ENTRIES = 16384,
237 MAX_CTRL_TXQ_ENTRIES = 1024,
238 MAX_RSPQ_ENTRIES = 16384,
239 MAX_RX_BUFFERS = 16384,
240 MAX_RX_JUMBO_BUFFERS = 16384,
242 MIN_CTRL_TXQ_ENTRIES = 4,
243 MIN_RSPQ_ENTRIES = 32,
245 MIN_FL_JUMBO_ENTRIES = 32
260 u32 report_filter_id:1;
268 enum { FILTER_NO_VLAN_PRI = 7 };
270 #define EEPROM_MAGIC 0x38E2F10C
272 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
274 /* Table for probing the cards. The desc field isn't actually used */
280 } cxgb_identifiers[] = {
281 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
282 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
283 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
284 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
285 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
286 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
287 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
288 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
289 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
290 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
291 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
292 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
293 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
294 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
298 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
302 t3rev2char(struct adapter *adapter)
306 switch(adapter->params.rev) {
321 static struct cxgb_ident *
322 cxgb_get_ident(device_t dev)
324 struct cxgb_ident *id;
326 for (id = cxgb_identifiers; id->desc != NULL; id++) {
327 if ((id->vendor == pci_get_vendor(dev)) &&
328 (id->device == pci_get_device(dev))) {
335 static const struct adapter_info *
336 cxgb_get_adapter_info(device_t dev)
338 struct cxgb_ident *id;
339 const struct adapter_info *ai;
341 id = cxgb_get_ident(dev);
345 ai = t3_get_adapter_info(id->index);
351 cxgb_controller_probe(device_t dev)
353 const struct adapter_info *ai;
354 char *ports, buf[80];
357 ai = cxgb_get_adapter_info(dev);
361 nports = ai->nports0 + ai->nports1;
367 snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
368 device_set_desc_copy(dev, buf);
369 return (BUS_PROBE_DEFAULT);
372 #define FW_FNAME "cxgb_t3fw"
373 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
374 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
377 upgrade_fw(adapter_t *sc)
379 const struct firmware *fw;
383 if ((fw = firmware_get(FW_FNAME)) == NULL) {
384 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
387 device_printf(sc->dev, "installing firmware on card\n");
388 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
391 device_printf(sc->dev, "failed to install firmware: %d\n",
394 t3_get_fw_version(sc, &vers);
395 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
396 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
397 G_FW_VERSION_MICRO(vers));
400 firmware_put(fw, FIRMWARE_UNLOAD);
406 * The cxgb_controller_attach function is responsible for the initial
407 * bringup of the device. Its responsibilities include:
409 * 1. Determine if the device supports MSI or MSI-X.
410 * 2. Allocate bus resources so that we can access the Base Address Register
411 * 3. Create and initialize mutexes for the controller and its control
412 * logic such as SGE and MDIO.
413 * 4. Call hardware specific setup routine for the adapter as a whole.
414 * 5. Allocate the BAR for doing MSI-X.
415 * 6. Setup the line interrupt iff MSI-X is not supported.
416 * 7. Create the driver's taskq.
417 * 8. Start one task queue service thread.
418 * 9. Check if the firmware and SRAM are up-to-date. They will be
419 * auto-updated later (before FULL_INIT_DONE), if required.
420 * 10. Create a child device for each MAC (port)
421 * 11. Initialize T3 private state.
422 * 12. Trigger the LED
423 * 13. Setup offload iff supported.
424 * 14. Reset/restart the tick callout.
427 * NOTE: Any modification or deviation from this list MUST be reflected in
428 * the above comment. Failure to do so will result in problems on various
429 * error conditions including link flapping.
432 cxgb_controller_attach(device_t dev)
435 const struct adapter_info *ai;
443 sc = device_get_softc(dev);
446 ai = cxgb_get_adapter_info(dev);
448 /* find the PCIe link width and set max read request to 4KB*/
449 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
452 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
453 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
454 if (sc->link_width < 8 &&
455 (ai->caps & SUPPORTED_10000baseT_Full)) {
456 device_printf(sc->dev,
457 "PCIe x%d Link, expect reduced performance\n",
461 pci_set_max_read_req(dev, 4096);
465 pci_enable_busmaster(dev);
467 * Allocate the registers and make them available to the driver.
468 * The registers that we care about for NIC mode are in BAR 0
470 sc->regs_rid = PCIR_BAR(0);
471 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
472 &sc->regs_rid, RF_ACTIVE)) == NULL) {
473 device_printf(dev, "Cannot allocate BAR region 0\n");
477 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
478 device_get_unit(dev));
479 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
481 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
482 device_get_unit(dev));
483 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
484 device_get_unit(dev));
485 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
486 device_get_unit(dev));
488 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
489 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
490 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
492 sc->bt = rman_get_bustag(sc->regs_res);
493 sc->bh = rman_get_bushandle(sc->regs_res);
494 sc->mmio_len = rman_get_size(sc->regs_res);
496 for (i = 0; i < MAX_NPORTS; i++)
497 sc->port[i].adapter = sc;
499 if (t3_prep_adapter(sc, ai, 1) < 0) {
500 printf("prep adapter failed\n");
505 sc->udbs_rid = PCIR_BAR(2);
507 if (is_offload(sc) &&
508 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
509 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
510 device_printf(dev, "Cannot allocate BAR region 1\n");
515 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
516 * enough messages for the queue sets. If that fails, try falling
517 * back to MSI. If that fails, then try falling back to the legacy
518 * interrupt pin model.
520 sc->msix_regs_rid = 0x20;
521 if ((msi_allowed >= 2) &&
522 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
523 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
526 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
527 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
529 if (pci_msix_count(dev) == 0 ||
530 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
531 sc->msi_count != msi_needed) {
532 device_printf(dev, "alloc msix failed - "
533 "msi_count=%d, msi_needed=%d, err=%d; "
534 "will try MSI\n", sc->msi_count,
538 pci_release_msi(dev);
539 bus_release_resource(dev, SYS_RES_MEMORY,
540 sc->msix_regs_rid, sc->msix_regs_res);
541 sc->msix_regs_res = NULL;
543 sc->flags |= USING_MSIX;
544 sc->cxgb_intr = cxgb_async_intr;
546 "using MSI-X interrupts (%u vectors)\n",
551 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
553 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
554 device_printf(dev, "alloc msi failed - "
555 "err=%d; will try INTx\n", error);
558 pci_release_msi(dev);
560 sc->flags |= USING_MSI;
561 sc->cxgb_intr = t3_intr_msi;
562 device_printf(dev, "using MSI interrupts\n");
565 if (sc->msi_count == 0) {
566 device_printf(dev, "using line interrupts\n");
567 sc->cxgb_intr = t3b_intr;
570 /* Create a private taskqueue thread for handling driver events */
571 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
572 taskqueue_thread_enqueue, &sc->tq);
573 if (sc->tq == NULL) {
574 device_printf(dev, "failed to allocate controller task queue\n");
578 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
579 device_get_nameunit(dev));
580 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
583 /* Create a periodic callout for checking adapter status */
584 callout_init(&sc->cxgb_tick_ch, TRUE);
586 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
588 * Warn user that a firmware update will be attempted in init.
590 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
591 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
592 sc->flags &= ~FW_UPTODATE;
594 sc->flags |= FW_UPTODATE;
597 if (t3_check_tpsram_version(sc) < 0) {
599 * Warn user that a firmware update will be attempted in init.
601 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
602 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
603 sc->flags &= ~TPS_UPTODATE;
605 sc->flags |= TPS_UPTODATE;
609 * Create a child device for each MAC. The ethernet attachment
610 * will be done in these children.
612 for (i = 0; i < (sc)->params.nports; i++) {
613 struct port_info *pi;
615 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
616 device_printf(dev, "failed to add child port\n");
622 pi->nqsets = port_qsets;
623 pi->first_qset = i*port_qsets;
625 pi->tx_chan = i >= ai->nports0;
626 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
627 sc->rxpkt_map[pi->txpkt_intf] = i;
628 sc->port[i].tx_chan = i >= ai->nports0;
629 sc->portdev[i] = child;
630 device_set_softc(child, pi);
632 if ((error = bus_generic_attach(dev)) != 0)
635 /* initialize sge private state */
636 t3_sge_init_adapter(sc);
641 if (is_offload(sc)) {
642 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
643 cxgb_adapter_ofld(sc);
645 error = t3_get_fw_version(sc, &vers);
649 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
650 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
651 G_FW_VERSION_MICRO(vers));
653 snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
654 ai->desc, is_offload(sc) ? "R" : "",
655 sc->params.vpd.ec, sc->params.vpd.sn);
656 device_set_desc_copy(dev, buf);
658 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
659 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
660 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
662 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
663 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
664 t3_add_attach_sysctls(sc);
667 error = cxgb_setup_interrupts(sc);
676 * The cxgb_controller_detach routine is called with the device is
677 * unloaded from the system.
681 cxgb_controller_detach(device_t dev)
685 sc = device_get_softc(dev);
693 * The cxgb_free() is called by the cxgb_controller_detach() routine
694 * to tear down the structures that were built up in
695 * cxgb_controller_attach(), and should be the final piece of work
696 * done when fully unloading the driver.
699 * 1. Shutting down the threads started by the cxgb_controller_attach()
701 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
702 * 3. Detaching all of the port devices created during the
703 * cxgb_controller_attach() routine.
704 * 4. Removing the device children created via cxgb_controller_attach().
705 * 5. Releasing PCI resources associated with the device.
706 * 6. Turning off the offload support, iff it was turned on.
707 * 7. Destroying the mutexes created in cxgb_controller_attach().
711 cxgb_free(struct adapter *sc)
716 sc->flags |= CXGB_SHUTDOWN;
720 * Make sure all child devices are gone.
722 bus_generic_detach(sc->dev);
723 for (i = 0; i < (sc)->params.nports; i++) {
724 if (sc->portdev[i] &&
725 device_delete_child(sc->dev, sc->portdev[i]) != 0)
726 device_printf(sc->dev, "failed to delete child port\n");
727 nqsets += sc->port[i].nqsets;
731 * At this point, it is as if cxgb_port_detach has run on all ports, and
732 * cxgb_down has run on the adapter. All interrupts have been silenced,
733 * all open devices have been closed.
735 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
736 __func__, sc->open_device_map));
737 for (i = 0; i < sc->params.nports; i++) {
738 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
743 * Finish off the adapter's callouts.
745 callout_drain(&sc->cxgb_tick_ch);
746 callout_drain(&sc->sge_timer_ch);
749 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
750 * sysctls are cleaned up by the kernel linker.
752 if (sc->flags & FULL_INIT_DONE) {
753 t3_free_sge_resources(sc, nqsets);
754 sc->flags &= ~FULL_INIT_DONE;
758 * Release all interrupt resources.
760 cxgb_teardown_interrupts(sc);
761 if (sc->flags & (USING_MSI | USING_MSIX)) {
762 device_printf(sc->dev, "releasing msi message(s)\n");
763 pci_release_msi(sc->dev);
765 device_printf(sc->dev, "no msi message to release\n");
768 if (sc->msix_regs_res != NULL) {
769 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
774 * Free the adapter's taskqueue.
776 if (sc->tq != NULL) {
777 taskqueue_free(sc->tq);
781 if (is_offload(sc)) {
782 clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
783 cxgb_adapter_unofld(sc);
787 if (sc->flags & CXGB_OFLD_INIT)
788 cxgb_offload_deactivate(sc);
790 free(sc->filters, M_DEVBUF);
795 if (sc->udbs_res != NULL)
796 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
799 if (sc->regs_res != NULL)
800 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
803 MTX_DESTROY(&sc->mdio_lock);
804 MTX_DESTROY(&sc->sge.reg_lock);
805 MTX_DESTROY(&sc->elmer_lock);
806 ADAPTER_LOCK_DEINIT(sc);
810 * setup_sge_qsets - configure SGE Tx/Rx/response queues
811 * @sc: the controller softc
813 * Determines how many sets of SGE queues to use and initializes them.
814 * We support multiple queue sets per port if we have MSI-X, otherwise
815 * just one queue set per port.
818 setup_sge_qsets(adapter_t *sc)
820 int i, j, err, irq_idx = 0, qset_idx = 0;
821 u_int ntxq = SGE_TXQ_PER_SET;
823 if ((err = t3_sge_alloc(sc)) != 0) {
824 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
828 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
831 for (i = 0; i < (sc)->params.nports; i++) {
832 struct port_info *pi = &sc->port[i];
834 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
835 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
836 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
837 &sc->params.sge.qset[qset_idx], ntxq, pi);
839 t3_free_sge_resources(sc, qset_idx);
840 device_printf(sc->dev,
841 "t3_sge_alloc_qset failed with %d\n", err);
851 cxgb_teardown_interrupts(adapter_t *sc)
855 for (i = 0; i < SGE_QSETS; i++) {
856 if (sc->msix_intr_tag[i] == NULL) {
858 /* Should have been setup fully or not at all */
859 KASSERT(sc->msix_irq_res[i] == NULL &&
860 sc->msix_irq_rid[i] == 0,
861 ("%s: half-done interrupt (%d).", __func__, i));
866 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
867 sc->msix_intr_tag[i]);
868 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
869 sc->msix_irq_res[i]);
871 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
872 sc->msix_irq_rid[i] = 0;
876 KASSERT(sc->irq_res != NULL,
877 ("%s: half-done interrupt.", __func__));
879 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
880 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
883 sc->irq_res = sc->intr_tag = NULL;
889 cxgb_setup_interrupts(adapter_t *sc)
891 struct resource *res;
893 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
895 sc->irq_rid = intr_flag ? 1 : 0;
896 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
897 RF_SHAREABLE | RF_ACTIVE);
898 if (sc->irq_res == NULL) {
899 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
900 intr_flag, sc->irq_rid);
904 err = bus_setup_intr(sc->dev, sc->irq_res,
905 INTR_MPSAFE | INTR_TYPE_NET, NULL,
906 sc->cxgb_intr, sc, &sc->intr_tag);
909 device_printf(sc->dev,
910 "Cannot set up interrupt (%x, %u, %d)\n",
911 intr_flag, sc->irq_rid, err);
912 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
914 sc->irq_res = sc->intr_tag = NULL;
919 /* That's all for INTx or MSI */
920 if (!(intr_flag & USING_MSIX) || err)
923 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
924 for (i = 0; i < sc->msi_count - 1; i++) {
926 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
927 RF_SHAREABLE | RF_ACTIVE);
929 device_printf(sc->dev, "Cannot allocate interrupt "
930 "for message %d\n", rid);
935 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
936 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
938 device_printf(sc->dev, "Cannot set up interrupt "
939 "for message %d (%d)\n", rid, err);
940 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
944 sc->msix_irq_rid[i] = rid;
945 sc->msix_irq_res[i] = res;
946 sc->msix_intr_tag[i] = tag;
947 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
951 cxgb_teardown_interrupts(sc);
958 cxgb_port_probe(device_t dev)
964 p = device_get_softc(dev);
966 snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
967 device_set_desc_copy(dev, buf);
973 cxgb_makedev(struct port_info *pi)
976 pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
977 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
979 if (pi->port_cdev == NULL)
982 pi->port_cdev->si_drv1 = (void *)pi;
987 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
988 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
989 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE)
990 #define CXGB_CAP_ENABLE (CXGB_CAP & ~IFCAP_TSO6)
993 cxgb_port_attach(device_t dev)
1000 p = device_get_softc(dev);
1002 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1003 device_get_unit(device_get_parent(dev)), p->port_id);
1004 PORT_LOCK_INIT(p, p->lockbuf);
1006 callout_init(&p->link_check_ch, CALLOUT_MPSAFE);
1007 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1009 /* Allocate an ifnet object and set it up */
1010 ifp = p->ifp = if_alloc(IFT_ETHER);
1012 device_printf(dev, "Cannot allocate ifnet\n");
1016 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1017 ifp->if_init = cxgb_init;
1019 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1020 ifp->if_ioctl = cxgb_ioctl;
1021 ifp->if_transmit = cxgb_transmit;
1022 ifp->if_qflush = cxgb_qflush;
1024 ifp->if_capabilities = CXGB_CAP;
1025 ifp->if_capenable = CXGB_CAP_ENABLE;
1026 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
1029 * Disable TSO on 4-port - it isn't supported by the firmware.
1031 if (sc->params.nports > 2) {
1032 ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1033 ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1034 ifp->if_hwassist &= ~CSUM_TSO;
1037 ether_ifattach(ifp, p->hw_addr);
1039 #ifdef DEFAULT_JUMBO
1040 if (sc->params.nports <= 2)
1041 ifp->if_mtu = ETHERMTU_JUMBO;
1043 if ((err = cxgb_makedev(p)) != 0) {
1044 printf("makedev failed %d\n", err);
1048 /* Create a list of media supported by this port */
1049 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1051 cxgb_build_medialist(p);
1053 t3_sge_init_port(p);
1059 * cxgb_port_detach() is called via the device_detach methods when
1060 * cxgb_free() calls the bus_generic_detach. It is responsible for
1061 * removing the device from the view of the kernel, i.e. from all
1062 * interfaces lists etc. This routine is only called when the driver is
1063 * being unloaded, not when the link goes down.
1066 cxgb_port_detach(device_t dev)
1068 struct port_info *p;
1072 p = device_get_softc(dev);
1075 /* Tell cxgb_ioctl and if_init that the port is going away */
1080 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1084 if (p->port_cdev != NULL)
1085 destroy_dev(p->port_cdev);
1087 cxgb_uninit_synchronized(p);
1088 ether_ifdetach(p->ifp);
1090 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1091 struct sge_qset *qs = &sc->sge.qs[i];
1092 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1094 callout_drain(&txq->txq_watchdog);
1095 callout_drain(&txq->txq_timer);
1098 PORT_LOCK_DEINIT(p);
1104 wakeup_one(&sc->flags);
1110 t3_fatal_err(struct adapter *sc)
1114 if (sc->flags & FULL_INIT_DONE) {
1116 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1117 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1118 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1119 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1120 t3_intr_disable(sc);
1122 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1123 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1124 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1125 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1129 t3_os_find_pci_capability(adapter_t *sc, int cap)
1132 struct pci_devinfo *dinfo;
1138 dinfo = device_get_ivars(dev);
1141 status = pci_read_config(dev, PCIR_STATUS, 2);
1142 if (!(status & PCIM_STATUS_CAPPRESENT))
1145 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1151 ptr = PCIR_CAP_PTR_2;
1157 ptr = pci_read_config(dev, ptr, 1);
1160 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1162 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1169 t3_os_pci_save_state(struct adapter *sc)
1172 struct pci_devinfo *dinfo;
1175 dinfo = device_get_ivars(dev);
1177 pci_cfg_save(dev, dinfo, 0);
1182 t3_os_pci_restore_state(struct adapter *sc)
1185 struct pci_devinfo *dinfo;
1188 dinfo = device_get_ivars(dev);
1190 pci_cfg_restore(dev, dinfo);
1195 * t3_os_link_changed - handle link status changes
1196 * @sc: the adapter associated with the link change
1197 * @port_id: the port index whose link status has changed
1198 * @link_status: the new status of the link
1199 * @speed: the new speed setting
1200 * @duplex: the new duplex setting
1201 * @fc: the new flow-control setting
1203 * This is the OS-dependent handler for link status changes. The OS
1204 * neutral handler takes care of most of the processing for these events,
1205 * then calls this handler for any OS-specific processing.
1208 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1209 int duplex, int fc, int mac_was_reset)
1211 struct port_info *pi = &adapter->port[port_id];
1212 struct ifnet *ifp = pi->ifp;
1214 /* no race with detach, so ifp should always be good */
1215 KASSERT(ifp, ("%s: if detached.", __func__));
1217 /* Reapply mac settings if they were lost due to a reset */
1218 if (mac_was_reset) {
1220 cxgb_update_mac_settings(pi);
1225 ifp->if_baudrate = IF_Mbps(speed);
1226 if_link_state_change(ifp, LINK_STATE_UP);
1228 if_link_state_change(ifp, LINK_STATE_DOWN);
1232 * t3_os_phymod_changed - handle PHY module changes
1233 * @phy: the PHY reporting the module change
1234 * @mod_type: new module type
1236 * This is the OS-dependent handler for PHY module changes. It is
1237 * invoked when a PHY module is removed or inserted for any OS-specific
1240 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1242 static const char *mod_str[] = {
1243 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1245 struct port_info *pi = &adap->port[port_id];
1246 int mod = pi->phy.modtype;
1248 if (mod != pi->media.ifm_cur->ifm_data)
1249 cxgb_build_medialist(pi);
1251 if (mod == phy_modtype_none)
1252 if_printf(pi->ifp, "PHY module unplugged\n");
1254 KASSERT(mod < ARRAY_SIZE(mod_str),
1255 ("invalid PHY module type %d", mod));
1256 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1261 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1265 * The ifnet might not be allocated before this gets called,
1266 * as this is called early on in attach by t3_prep_adapter
1267 * save the address off in the port structure
1270 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1271 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1275 * Programs the XGMAC based on the settings in the ifnet. These settings
1276 * include MTU, MAC address, mcast addresses, etc.
1279 cxgb_update_mac_settings(struct port_info *p)
1281 struct ifnet *ifp = p->ifp;
1282 struct t3_rx_mode rm;
1283 struct cmac *mac = &p->mac;
1286 PORT_LOCK_ASSERT_OWNED(p);
1288 bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1291 if (ifp->if_capenable & IFCAP_VLAN_MTU)
1292 mtu += ETHER_VLAN_ENCAP_LEN;
1294 hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1296 t3_mac_set_mtu(mac, mtu);
1297 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1298 t3_mac_set_address(mac, 0, p->hw_addr);
1299 t3_init_rx_mode(&rm, p);
1300 t3_mac_set_rx_mode(mac, &rm);
1305 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1310 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1319 init_tp_parity(struct adapter *adap)
1323 struct cpl_set_tcb_field *greq;
1324 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1326 t3_tp_set_offload_mode(adap, 1);
1328 for (i = 0; i < 16; i++) {
1329 struct cpl_smt_write_req *req;
1331 m = m_gethdr(M_WAITOK, MT_DATA);
1332 req = mtod(m, struct cpl_smt_write_req *);
1333 m->m_len = m->m_pkthdr.len = sizeof(*req);
1334 memset(req, 0, sizeof(*req));
1335 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1336 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1338 t3_mgmt_tx(adap, m);
1341 for (i = 0; i < 2048; i++) {
1342 struct cpl_l2t_write_req *req;
1344 m = m_gethdr(M_WAITOK, MT_DATA);
1345 req = mtod(m, struct cpl_l2t_write_req *);
1346 m->m_len = m->m_pkthdr.len = sizeof(*req);
1347 memset(req, 0, sizeof(*req));
1348 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1349 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1350 req->params = htonl(V_L2T_W_IDX(i));
1351 t3_mgmt_tx(adap, m);
1354 for (i = 0; i < 2048; i++) {
1355 struct cpl_rte_write_req *req;
1357 m = m_gethdr(M_WAITOK, MT_DATA);
1358 req = mtod(m, struct cpl_rte_write_req *);
1359 m->m_len = m->m_pkthdr.len = sizeof(*req);
1360 memset(req, 0, sizeof(*req));
1361 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1362 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1363 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1364 t3_mgmt_tx(adap, m);
1367 m = m_gethdr(M_WAITOK, MT_DATA);
1368 greq = mtod(m, struct cpl_set_tcb_field *);
1369 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1370 memset(greq, 0, sizeof(*greq));
1371 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1372 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1373 greq->mask = htobe64(1);
1374 t3_mgmt_tx(adap, m);
1376 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1377 t3_tp_set_offload_mode(adap, 0);
1382 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1383 * @adap: the adapter
1385 * Sets up RSS to distribute packets to multiple receive queues. We
1386 * configure the RSS CPU lookup table to distribute to the number of HW
1387 * receive queues, and the response queue lookup table to narrow that
1388 * down to the response queues actually configured for each port.
1389 * We always configure the RSS mapping for two ports since the mapping
1390 * table has plenty of entries.
1393 setup_rss(adapter_t *adap)
1397 uint8_t cpus[SGE_QSETS + 1];
1398 uint16_t rspq_map[RSS_TABLE_SIZE];
1400 for (i = 0; i < SGE_QSETS; ++i)
1402 cpus[SGE_QSETS] = 0xff;
1405 for_each_port(adap, i) {
1406 const struct port_info *pi = adap2pinfo(adap, i);
1408 nq[pi->tx_chan] += pi->nqsets;
1410 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1411 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1412 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1415 /* Calculate the reverse RSS map table */
1416 for (i = 0; i < SGE_QSETS; ++i)
1417 adap->rrss_map[i] = 0xff;
1418 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1419 if (adap->rrss_map[rspq_map[i]] == 0xff)
1420 adap->rrss_map[rspq_map[i]] = i;
1422 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1423 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1424 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1430 * Sends an mbuf to an offload queue driver
1431 * after dealing with any active network taps.
1434 offload_tx(struct t3cdev *tdev, struct mbuf *m)
1438 ret = t3_offload_tx(tdev, m);
1443 write_smt_entry(struct adapter *adapter, int idx)
1445 struct port_info *pi = &adapter->port[idx];
1446 struct cpl_smt_write_req *req;
1449 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1452 req = mtod(m, struct cpl_smt_write_req *);
1453 m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
1455 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1456 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1457 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
1459 memset(req->src_mac1, 0, sizeof(req->src_mac1));
1460 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1462 m_set_priority(m, 1);
1464 offload_tx(&adapter->tdev, m);
1470 init_smt(struct adapter *adapter)
1474 for_each_port(adapter, i)
1475 write_smt_entry(adapter, i);
1480 init_port_mtus(adapter_t *adapter)
1482 unsigned int mtus = ETHERMTU | (ETHERMTU << 16);
1484 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1488 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1492 struct mngt_pktsched_wr *req;
1494 m = m_gethdr(M_DONTWAIT, MT_DATA);
1496 req = mtod(m, struct mngt_pktsched_wr *);
1497 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1498 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1503 req->binding = port;
1504 m->m_len = m->m_pkthdr.len = sizeof(*req);
1505 t3_mgmt_tx(adap, m);
1510 bind_qsets(adapter_t *sc)
1514 for (i = 0; i < (sc)->params.nports; ++i) {
1515 const struct port_info *pi = adap2pinfo(sc, i);
1517 for (j = 0; j < pi->nqsets; ++j) {
1518 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1526 update_tpeeprom(struct adapter *adap)
1528 const struct firmware *tpeeprom;
1531 unsigned int major, minor;
1535 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1537 major = G_TP_VERSION_MAJOR(version);
1538 minor = G_TP_VERSION_MINOR(version);
1539 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1542 rev = t3rev2char(adap);
1543 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1545 tpeeprom = firmware_get(name);
1546 if (tpeeprom == NULL) {
1547 device_printf(adap->dev,
1548 "could not load TP EEPROM: unable to load %s\n",
1553 len = tpeeprom->datasize - 4;
1555 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1557 goto release_tpeeprom;
1559 if (len != TP_SRAM_LEN) {
1560 device_printf(adap->dev,
1561 "%s length is wrong len=%d expected=%d\n", name,
1566 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1570 device_printf(adap->dev,
1571 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1572 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1574 device_printf(adap->dev,
1575 "Protocol SRAM image update in EEPROM failed\n");
1578 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1584 update_tpsram(struct adapter *adap)
1586 const struct firmware *tpsram;
1590 rev = t3rev2char(adap);
1591 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1593 update_tpeeprom(adap);
1595 tpsram = firmware_get(name);
1596 if (tpsram == NULL){
1597 device_printf(adap->dev, "could not load TP SRAM\n");
1600 device_printf(adap->dev, "updating TP SRAM\n");
1602 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1604 goto release_tpsram;
1606 ret = t3_set_proto_sram(adap, tpsram->data);
1608 device_printf(adap->dev, "loading protocol SRAM failed\n");
1611 firmware_put(tpsram, FIRMWARE_UNLOAD);
1617 * cxgb_up - enable the adapter
1618 * @adap: adapter being enabled
1620 * Called when the first port is enabled, this function performs the
1621 * actions necessary to make an adapter operational, such as completing
1622 * the initialization of HW modules, and enabling interrupts.
1625 cxgb_up(struct adapter *sc)
1628 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1630 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1631 __func__, sc->open_device_map));
1633 if ((sc->flags & FULL_INIT_DONE) == 0) {
1635 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1637 if ((sc->flags & FW_UPTODATE) == 0)
1638 if ((err = upgrade_fw(sc)))
1641 if ((sc->flags & TPS_UPTODATE) == 0)
1642 if ((err = update_tpsram(sc)))
1645 if (is_offload(sc) && nfilters != 0) {
1646 sc->params.mc5.nservers = 0;
1649 sc->params.mc5.nfilters = mxf;
1651 sc->params.mc5.nfilters = min(nfilters, mxf);
1654 err = t3_init_hw(sc, 0);
1658 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1659 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1661 err = setup_sge_qsets(sc);
1668 t3_add_configured_sysctls(sc);
1669 sc->flags |= FULL_INIT_DONE;
1676 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1677 is_offload(sc) && init_tp_parity(sc) == 0)
1678 sc->flags |= TP_PARITY_INIT;
1680 if (sc->flags & TP_PARITY_INIT) {
1681 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1682 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1685 if (!(sc->flags & QUEUES_BOUND)) {
1687 setup_hw_filters(sc);
1688 sc->flags |= QUEUES_BOUND;
1691 t3_sge_reset_adapter(sc);
1697 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1698 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1699 * during controller_detach, not here.
1702 cxgb_down(struct adapter *sc)
1705 t3_intr_disable(sc);
1709 offload_open(struct port_info *pi)
1711 struct adapter *sc = pi->adapter;
1712 struct t3cdev *tdev = &sc->tdev;
1714 setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT);
1716 t3_tp_set_offload_mode(sc, 1);
1717 tdev->lldev = pi->ifp;
1719 t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd,
1720 sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff);
1722 cxgb_add_clients(tdev);
1728 offload_close(struct t3cdev *tdev)
1730 struct adapter *adapter = tdev2adap(tdev);
1732 if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1735 /* Call back all registered clients */
1736 cxgb_remove_clients(tdev);
1739 cxgb_set_dummy_ops(tdev);
1740 t3_tp_set_offload_mode(adapter, 0);
1742 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1748 * if_init for cxgb ports.
1751 cxgb_init(void *arg)
1753 struct port_info *p = arg;
1754 struct adapter *sc = p->adapter;
1757 cxgb_init_locked(p); /* releases adapter lock */
1758 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1762 cxgb_init_locked(struct port_info *p)
1764 struct adapter *sc = p->adapter;
1765 struct ifnet *ifp = p->ifp;
1766 struct cmac *mac = &p->mac;
1767 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1769 ADAPTER_LOCK_ASSERT_OWNED(sc);
1771 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1773 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1782 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1785 * The code that runs during one-time adapter initialization can sleep
1786 * so it's important not to hold any locks across it.
1788 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1796 if (sc->open_device_map == 0) {
1797 if ((rc = cxgb_up(sc)) != 0)
1800 if (is_offload(sc) && !ofld_disable && offload_open(p))
1802 "Could not initialize offload capabilities\n");
1806 if (isset(&sc->open_device_map, p->port_id) &&
1807 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1811 t3_port_intr_enable(sc, p->port_id);
1812 if (!mac->multiport)
1814 cxgb_update_mac_settings(p);
1815 t3_link_start(&p->phy, mac, &p->link_config);
1816 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1817 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1818 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1821 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1822 struct sge_qset *qs = &sc->sge.qs[i];
1823 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1825 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1826 txq->txq_watchdog.c_cpu);
1830 setbit(&sc->open_device_map, p->port_id);
1831 callout_reset(&p->link_check_ch,
1832 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1833 link_check_callout, p);
1838 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1842 wakeup_one(&sc->flags);
1848 cxgb_uninit_locked(struct port_info *p)
1850 struct adapter *sc = p->adapter;
1853 ADAPTER_LOCK_ASSERT_OWNED(sc);
1855 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1856 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1865 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1869 rc = cxgb_uninit_synchronized(p);
1872 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1874 wakeup_one(&sc->flags);
1881 * Called on "ifconfig down", and from port_detach
1884 cxgb_uninit_synchronized(struct port_info *pi)
1886 struct adapter *sc = pi->adapter;
1887 struct ifnet *ifp = pi->ifp;
1890 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1892 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1895 * Clear this port's bit from the open device map, and then drain all
1896 * the tasks that can access/manipulate this port's port_info or ifp.
1897 * We disable this port's interrupts here and so the slow/ext
1898 * interrupt tasks won't be enqueued. The tick task will continue to
1899 * be enqueued every second but the runs after this drain will not see
1900 * this port in the open device map.
1902 * A well behaved task must take open_device_map into account and ignore
1903 * ports that are not open.
1905 clrbit(&sc->open_device_map, pi->port_id);
1906 t3_port_intr_disable(sc, pi->port_id);
1907 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1908 taskqueue_drain(sc->tq, &sc->tick_task);
1910 callout_drain(&pi->link_check_ch);
1911 taskqueue_drain(sc->tq, &pi->link_check_task);
1914 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1916 /* disable pause frames */
1917 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1919 /* Reset RX FIFO HWM */
1920 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1921 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1925 /* Wait for TXFIFO empty */
1926 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1927 F_TXFIFO_EMPTY, 1, 20, 5);
1930 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1933 pi->phy.ops->power_down(&pi->phy, 1);
1937 pi->link_config.link_ok = 0;
1938 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1940 if ((sc->open_device_map & PORT_MASK) == 0)
1941 offload_close(&sc->tdev);
1943 if (sc->open_device_map == 0)
1944 cxgb_down(pi->adapter);
1950 * Mark lro enabled or disabled in all qsets for this port
1953 cxgb_set_lro(struct port_info *p, int enabled)
1956 struct adapter *adp = p->adapter;
1959 for (i = 0; i < p->nqsets; i++) {
1960 q = &adp->sge.qs[p->first_qset + i];
1961 q->lro.enabled = (enabled != 0);
1967 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1969 struct port_info *p = ifp->if_softc;
1970 struct adapter *sc = p->adapter;
1971 struct ifreq *ifr = (struct ifreq *)data;
1972 int flags, error = 0, mtu;
1978 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1986 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1991 cxgb_update_mac_settings(p);
2002 if (ifp->if_flags & IFF_UP) {
2003 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2004 flags = p->if_flags;
2005 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
2006 ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
2012 cxgb_update_mac_settings(p);
2017 error = cxgb_init_locked(p);
2018 p->if_flags = ifp->if_flags;
2019 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2020 error = cxgb_uninit_locked(p);
2024 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2029 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
2033 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2035 cxgb_update_mac_settings(p);
2043 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
2047 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2048 if (mask & IFCAP_TXCSUM) {
2049 ifp->if_capenable ^= IFCAP_TXCSUM;
2050 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
2052 if (IFCAP_TSO & ifp->if_capenable &&
2053 !(IFCAP_TXCSUM & ifp->if_capenable)) {
2054 ifp->if_capenable &= ~IFCAP_TSO;
2055 ifp->if_hwassist &= ~CSUM_TSO;
2057 "tso disabled due to -txcsum.\n");
2060 if (mask & IFCAP_RXCSUM)
2061 ifp->if_capenable ^= IFCAP_RXCSUM;
2062 if (mask & IFCAP_TSO4) {
2063 ifp->if_capenable ^= IFCAP_TSO4;
2065 if (IFCAP_TSO & ifp->if_capenable) {
2066 if (IFCAP_TXCSUM & ifp->if_capenable)
2067 ifp->if_hwassist |= CSUM_TSO;
2069 ifp->if_capenable &= ~IFCAP_TSO;
2070 ifp->if_hwassist &= ~CSUM_TSO;
2072 "enable txcsum first.\n");
2076 ifp->if_hwassist &= ~CSUM_TSO;
2078 if (mask & IFCAP_LRO) {
2079 ifp->if_capenable ^= IFCAP_LRO;
2081 /* Safe to do this even if cxgb_up not called yet */
2082 cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2084 if (mask & IFCAP_VLAN_HWTAGGING) {
2085 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2086 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2088 cxgb_update_mac_settings(p);
2092 if (mask & IFCAP_VLAN_MTU) {
2093 ifp->if_capenable ^= IFCAP_VLAN_MTU;
2094 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2096 cxgb_update_mac_settings(p);
2100 if (mask & IFCAP_VLAN_HWTSO)
2101 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2102 if (mask & IFCAP_VLAN_HWCSUM)
2103 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2105 #ifdef VLAN_CAPABILITIES
2106 VLAN_CAPABILITIES(ifp);
2112 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2115 error = ether_ioctl(ifp, command, data);
2122 cxgb_media_change(struct ifnet *ifp)
2124 return (EOPNOTSUPP);
2128 * Translates phy->modtype to the correct Ethernet media subtype.
2131 cxgb_ifm_type(int mod)
2134 case phy_modtype_sr:
2135 return (IFM_10G_SR);
2136 case phy_modtype_lr:
2137 return (IFM_10G_LR);
2138 case phy_modtype_lrm:
2139 return (IFM_10G_LRM);
2140 case phy_modtype_twinax:
2141 return (IFM_10G_TWINAX);
2142 case phy_modtype_twinax_long:
2143 return (IFM_10G_TWINAX_LONG);
2144 case phy_modtype_none:
2146 case phy_modtype_unknown:
2147 return (IFM_UNKNOWN);
2150 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2151 return (IFM_UNKNOWN);
2155 * Rebuilds the ifmedia list for this port, and sets the current media.
2158 cxgb_build_medialist(struct port_info *p)
2160 struct cphy *phy = &p->phy;
2161 struct ifmedia *media = &p->media;
2162 int mod = phy->modtype;
2163 int m = IFM_ETHER | IFM_FDX;
2167 ifmedia_removeall(media);
2168 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2171 if (phy->caps & SUPPORTED_10000baseT_Full)
2172 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2174 if (phy->caps & SUPPORTED_1000baseT_Full)
2175 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2177 if (phy->caps & SUPPORTED_100baseT_Full)
2178 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2180 if (phy->caps & SUPPORTED_10baseT_Full)
2181 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2183 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2184 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2186 } else if (phy->caps & SUPPORTED_TP) {
2189 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2190 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2192 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2193 ifmedia_set(media, m | IFM_10G_CX4);
2195 } else if (phy->caps & SUPPORTED_FIBRE &&
2196 phy->caps & SUPPORTED_10000baseT_Full) {
2197 /* 10G optical (but includes SFP+ twinax) */
2199 m |= cxgb_ifm_type(mod);
2200 if (IFM_SUBTYPE(m) == IFM_NONE)
2203 ifmedia_add(media, m, mod, NULL);
2204 ifmedia_set(media, m);
2206 } else if (phy->caps & SUPPORTED_FIBRE &&
2207 phy->caps & SUPPORTED_1000baseT_Full) {
2210 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2211 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2212 ifmedia_set(media, m | IFM_1000_SX);
2215 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2223 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2225 struct port_info *p = ifp->if_softc;
2226 struct ifmedia_entry *cur = p->media.ifm_cur;
2227 int speed = p->link_config.speed;
2229 if (cur->ifm_data != p->phy.modtype) {
2230 cxgb_build_medialist(p);
2231 cur = p->media.ifm_cur;
2234 ifmr->ifm_status = IFM_AVALID;
2235 if (!p->link_config.link_ok)
2238 ifmr->ifm_status |= IFM_ACTIVE;
2241 * active and current will differ iff current media is autoselect. That
2242 * can happen only for copper RJ45.
2244 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2246 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2247 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2249 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2250 if (speed == SPEED_10000)
2251 ifmr->ifm_active |= IFM_10G_T;
2252 else if (speed == SPEED_1000)
2253 ifmr->ifm_active |= IFM_1000_T;
2254 else if (speed == SPEED_100)
2255 ifmr->ifm_active |= IFM_100_TX;
2256 else if (speed == SPEED_10)
2257 ifmr->ifm_active |= IFM_10_T;
2259 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2264 cxgb_async_intr(void *data)
2266 adapter_t *sc = data;
2268 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2269 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2270 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2274 link_check_callout(void *arg)
2276 struct port_info *pi = arg;
2277 struct adapter *sc = pi->adapter;
2279 if (!isset(&sc->open_device_map, pi->port_id))
2282 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2286 check_link_status(void *arg, int pending)
2288 struct port_info *pi = arg;
2289 struct adapter *sc = pi->adapter;
2291 if (!isset(&sc->open_device_map, pi->port_id))
2294 t3_link_changed(sc, pi->port_id);
2296 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ))
2297 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2301 t3_os_link_intr(struct port_info *pi)
2304 * Schedule a link check in the near future. If the link is flapping
2305 * rapidly we'll keep resetting the callout and delaying the check until
2306 * things stabilize a bit.
2308 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2312 check_t3b2_mac(struct adapter *sc)
2316 if (sc->flags & CXGB_SHUTDOWN)
2319 for_each_port(sc, i) {
2320 struct port_info *p = &sc->port[i];
2323 struct ifnet *ifp = p->ifp;
2326 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2327 !p->link_config.link_ok)
2330 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2331 ("%s: state mismatch (drv_flags %x, device_map %x)",
2332 __func__, ifp->if_drv_flags, sc->open_device_map));
2335 status = t3b2_mac_watchdog_task(&p->mac);
2337 p->mac.stats.num_toggled++;
2338 else if (status == 2) {
2339 struct cmac *mac = &p->mac;
2341 cxgb_update_mac_settings(p);
2342 t3_link_start(&p->phy, mac, &p->link_config);
2343 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2344 t3_port_intr_enable(sc, p->port_id);
2345 p->mac.stats.num_resets++;
2352 cxgb_tick(void *arg)
2354 adapter_t *sc = (adapter_t *)arg;
2356 if (sc->flags & CXGB_SHUTDOWN)
2359 taskqueue_enqueue(sc->tq, &sc->tick_task);
2360 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2364 cxgb_tick_handler(void *arg, int count)
2366 adapter_t *sc = (adapter_t *)arg;
2367 const struct adapter_params *p = &sc->params;
2369 uint32_t cause, reset;
2371 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2374 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2377 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2379 struct sge_qset *qs = &sc->sge.qs[0];
2382 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2385 for (i = 0; i < SGE_QSETS; i++) {
2387 qs[i].rspq.starved++;
2391 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2393 for (i = 0; i < SGE_QSETS * 2; i++) {
2395 qs[i / 2].fl[i % 2].empty++;
2401 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2402 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2405 for (i = 0; i < sc->params.nports; i++) {
2406 struct port_info *pi = &sc->port[i];
2407 struct ifnet *ifp = pi->ifp;
2408 struct cmac *mac = &pi->mac;
2409 struct mac_stats *mstats = &mac->stats;
2412 if (!isset(&sc->open_device_map, pi->port_id))
2416 t3_mac_update_stats(mac);
2419 ifp->if_opackets = mstats->tx_frames;
2420 ifp->if_ipackets = mstats->rx_frames;
2421 ifp->if_obytes = mstats->tx_octets;
2422 ifp->if_ibytes = mstats->rx_octets;
2423 ifp->if_omcasts = mstats->tx_mcast_frames;
2424 ifp->if_imcasts = mstats->rx_mcast_frames;
2425 ifp->if_collisions = mstats->tx_total_collisions;
2426 ifp->if_iqdrops = mstats->rx_cong_drops;
2429 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets; j++)
2430 drops += sc->sge.qs[j].txq[TXQ_ETH].txq_mr->br_drops;
2431 ifp->if_snd.ifq_drops = drops;
2434 mstats->tx_excess_collisions +
2435 mstats->tx_underrun +
2436 mstats->tx_len_errs +
2437 mstats->tx_mac_internal_errs +
2438 mstats->tx_excess_deferral +
2439 mstats->tx_fcs_errs;
2442 mstats->rx_data_errs +
2443 mstats->rx_sequence_errs +
2445 mstats->rx_too_long +
2446 mstats->rx_mac_internal_errs +
2448 mstats->rx_fcs_errs;
2453 /* Count rx fifo overflows, once per second */
2454 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2456 if (cause & F_RXFIFO_OVERFLOW) {
2457 mac->stats.rx_fifo_ovfl++;
2458 reset |= F_RXFIFO_OVERFLOW;
2460 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2465 touch_bars(device_t dev)
2470 #if !defined(__LP64__) && 0
2473 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2474 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2475 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2476 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2477 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2478 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2483 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2487 u32 aligned_offset, aligned_len, *p;
2488 struct adapter *adapter = pi->adapter;
2491 aligned_offset = offset & ~3;
2492 aligned_len = (len + (offset & 3) + 3) & ~3;
2494 if (aligned_offset != offset || aligned_len != len) {
2495 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2498 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2499 if (!err && aligned_len > 4)
2500 err = t3_seeprom_read(adapter,
2501 aligned_offset + aligned_len - 4,
2502 (u32 *)&buf[aligned_len - 4]);
2505 memcpy(buf + (offset & 3), data, len);
2507 buf = (uint8_t *)(uintptr_t)data;
2509 err = t3_seeprom_wp(adapter, 0);
2513 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2514 err = t3_seeprom_write(adapter, aligned_offset, *p);
2515 aligned_offset += 4;
2519 err = t3_seeprom_wp(adapter, 1);
2522 free(buf, M_DEVBUF);
2528 in_range(int val, int lo, int hi)
2530 return val < 0 || (val <= hi && val >= lo);
2534 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2540 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2546 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2547 int fflag, struct thread *td)
2550 struct port_info *pi = dev->si_drv1;
2551 adapter_t *sc = pi->adapter;
2553 #ifdef PRIV_SUPPORTED
2554 if (priv_check(td, PRIV_DRIVER)) {
2556 printf("user does not have access to privileged ioctls\n");
2562 printf("user does not have access to privileged ioctls\n");
2568 case CHELSIO_GET_MIIREG: {
2570 struct cphy *phy = &pi->phy;
2571 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2573 if (!phy->mdio_read)
2574 return (EOPNOTSUPP);
2576 mmd = mid->phy_id >> 8;
2579 else if (mmd > MDIO_DEV_VEND2)
2582 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2583 mid->reg_num, &val);
2585 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2586 mid->reg_num & 0x1f, &val);
2591 case CHELSIO_SET_MIIREG: {
2592 struct cphy *phy = &pi->phy;
2593 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2595 if (!phy->mdio_write)
2596 return (EOPNOTSUPP);
2598 mmd = mid->phy_id >> 8;
2601 else if (mmd > MDIO_DEV_VEND2)
2604 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2605 mmd, mid->reg_num, mid->val_in);
2607 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2608 mid->reg_num & 0x1f,
2612 case CHELSIO_SETREG: {
2613 struct ch_reg *edata = (struct ch_reg *)data;
2614 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2616 t3_write_reg(sc, edata->addr, edata->val);
2619 case CHELSIO_GETREG: {
2620 struct ch_reg *edata = (struct ch_reg *)data;
2621 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2623 edata->val = t3_read_reg(sc, edata->addr);
2626 case CHELSIO_GET_SGE_CONTEXT: {
2627 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2628 mtx_lock_spin(&sc->sge.reg_lock);
2629 switch (ecntxt->cntxt_type) {
2630 case CNTXT_TYPE_EGRESS:
2631 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2635 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2638 case CNTXT_TYPE_RSP:
2639 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2643 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2650 mtx_unlock_spin(&sc->sge.reg_lock);
2653 case CHELSIO_GET_SGE_DESC: {
2654 struct ch_desc *edesc = (struct ch_desc *)data;
2656 if (edesc->queue_num >= SGE_QSETS * 6)
2658 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2659 edesc->queue_num % 6, edesc->idx, edesc->data);
2665 case CHELSIO_GET_QSET_PARAMS: {
2666 struct qset_params *q;
2667 struct ch_qset_params *t = (struct ch_qset_params *)data;
2668 int q1 = pi->first_qset;
2669 int nqsets = pi->nqsets;
2672 if (t->qset_idx >= nqsets)
2675 i = q1 + t->qset_idx;
2676 q = &sc->params.sge.qset[i];
2677 t->rspq_size = q->rspq_size;
2678 t->txq_size[0] = q->txq_size[0];
2679 t->txq_size[1] = q->txq_size[1];
2680 t->txq_size[2] = q->txq_size[2];
2681 t->fl_size[0] = q->fl_size;
2682 t->fl_size[1] = q->jumbo_size;
2683 t->polling = q->polling;
2685 t->intr_lat = q->coalesce_usecs;
2686 t->cong_thres = q->cong_thres;
2689 if ((sc->flags & FULL_INIT_DONE) == 0)
2691 else if (sc->flags & USING_MSIX)
2692 t->vector = rman_get_start(sc->msix_irq_res[i]);
2694 t->vector = rman_get_start(sc->irq_res);
2698 case CHELSIO_GET_QSET_NUM: {
2699 struct ch_reg *edata = (struct ch_reg *)data;
2700 edata->val = pi->nqsets;
2703 case CHELSIO_LOAD_FW: {
2706 struct ch_mem_range *t = (struct ch_mem_range *)data;
2709 * You're allowed to load a firmware only before FULL_INIT_DONE
2711 * FW_UPTODATE is also set so the rest of the initialization
2712 * will not overwrite what was loaded here. This gives you the
2713 * flexibility to load any firmware (and maybe shoot yourself in
2718 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2723 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2727 error = copyin(t->buf, fw_data, t->len);
2730 error = -t3_load_fw(sc, fw_data, t->len);
2732 if (t3_get_fw_version(sc, &vers) == 0) {
2733 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2734 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2735 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2739 sc->flags |= FW_UPTODATE;
2741 free(fw_data, M_DEVBUF);
2745 case CHELSIO_LOAD_BOOT: {
2747 struct ch_mem_range *t = (struct ch_mem_range *)data;
2749 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2753 error = copyin(t->buf, boot_data, t->len);
2755 error = -t3_load_boot(sc, boot_data, t->len);
2757 free(boot_data, M_DEVBUF);
2760 case CHELSIO_GET_PM: {
2761 struct ch_pm *m = (struct ch_pm *)data;
2762 struct tp_params *p = &sc->params.tp;
2764 if (!is_offload(sc))
2765 return (EOPNOTSUPP);
2767 m->tx_pg_sz = p->tx_pg_size;
2768 m->tx_num_pg = p->tx_num_pgs;
2769 m->rx_pg_sz = p->rx_pg_size;
2770 m->rx_num_pg = p->rx_num_pgs;
2771 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2775 case CHELSIO_SET_PM: {
2776 struct ch_pm *m = (struct ch_pm *)data;
2777 struct tp_params *p = &sc->params.tp;
2779 if (!is_offload(sc))
2780 return (EOPNOTSUPP);
2781 if (sc->flags & FULL_INIT_DONE)
2784 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2785 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2786 return (EINVAL); /* not power of 2 */
2787 if (!(m->rx_pg_sz & 0x14000))
2788 return (EINVAL); /* not 16KB or 64KB */
2789 if (!(m->tx_pg_sz & 0x1554000))
2791 if (m->tx_num_pg == -1)
2792 m->tx_num_pg = p->tx_num_pgs;
2793 if (m->rx_num_pg == -1)
2794 m->rx_num_pg = p->rx_num_pgs;
2795 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2797 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2798 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2801 p->rx_pg_size = m->rx_pg_sz;
2802 p->tx_pg_size = m->tx_pg_sz;
2803 p->rx_num_pgs = m->rx_num_pg;
2804 p->tx_num_pgs = m->tx_num_pg;
2807 case CHELSIO_SETMTUTAB: {
2808 struct ch_mtus *m = (struct ch_mtus *)data;
2811 if (!is_offload(sc))
2812 return (EOPNOTSUPP);
2813 if (offload_running(sc))
2815 if (m->nmtus != NMTUS)
2817 if (m->mtus[0] < 81) /* accommodate SACK */
2821 * MTUs must be in ascending order
2823 for (i = 1; i < NMTUS; ++i)
2824 if (m->mtus[i] < m->mtus[i - 1])
2827 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2830 case CHELSIO_GETMTUTAB: {
2831 struct ch_mtus *m = (struct ch_mtus *)data;
2833 if (!is_offload(sc))
2834 return (EOPNOTSUPP);
2836 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2840 case CHELSIO_GET_MEM: {
2841 struct ch_mem_range *t = (struct ch_mem_range *)data;
2847 * Use these to avoid modifying len/addr in the return
2850 uint32_t len = t->len, addr = t->addr;
2852 if (!is_offload(sc))
2853 return (EOPNOTSUPP);
2854 if (!(sc->flags & FULL_INIT_DONE))
2855 return (EIO); /* need the memory controllers */
2856 if ((addr & 0x7) || (len & 0x7))
2858 if (t->mem_id == MEM_CM)
2860 else if (t->mem_id == MEM_PMRX)
2862 else if (t->mem_id == MEM_PMTX)
2869 * bits 0..9: chip version
2870 * bits 10..15: chip revision
2872 t->version = 3 | (sc->params.rev << 10);
2875 * Read 256 bytes at a time as len can be large and we don't
2876 * want to use huge intermediate buffers.
2878 useraddr = (uint8_t *)t->buf;
2880 unsigned int chunk = min(len, sizeof(buf));
2882 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2885 if (copyout(buf, useraddr, chunk))
2893 case CHELSIO_READ_TCAM_WORD: {
2894 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2896 if (!is_offload(sc))
2897 return (EOPNOTSUPP);
2898 if (!(sc->flags & FULL_INIT_DONE))
2899 return (EIO); /* need MC5 */
2900 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2903 case CHELSIO_SET_TRACE_FILTER: {
2904 struct ch_trace *t = (struct ch_trace *)data;
2905 const struct trace_params *tp;
2907 tp = (const struct trace_params *)&t->sip;
2909 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2912 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2916 case CHELSIO_SET_PKTSCHED: {
2917 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2918 if (sc->open_device_map == 0)
2920 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2924 case CHELSIO_IFCONF_GETREGS: {
2925 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2926 int reglen = cxgb_get_regs_len();
2927 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2931 if (regs->len > reglen)
2933 else if (regs->len < reglen)
2937 cxgb_get_regs(sc, regs, buf);
2938 error = copyout(buf, regs->data, reglen);
2940 free(buf, M_DEVBUF);
2944 case CHELSIO_SET_HW_SCHED: {
2945 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2946 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2948 if ((sc->flags & FULL_INIT_DONE) == 0)
2949 return (EAGAIN); /* need TP to be initialized */
2950 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2951 !in_range(t->channel, 0, 1) ||
2952 !in_range(t->kbps, 0, 10000000) ||
2953 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2954 !in_range(t->flow_ipg, 0,
2955 dack_ticks_to_usec(sc, 0x7ff)))
2959 error = t3_config_sched(sc, t->kbps, t->sched);
2963 if (t->class_ipg >= 0)
2964 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2965 if (t->flow_ipg >= 0) {
2966 t->flow_ipg *= 1000; /* us -> ns */
2967 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2970 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2972 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2973 bit, t->mode ? bit : 0);
2975 if (t->channel >= 0)
2976 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2977 1 << t->sched, t->channel << t->sched);
2980 case CHELSIO_GET_EEPROM: {
2982 struct ch_eeprom *e = (struct ch_eeprom *)data;
2983 uint8_t *buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2988 e->magic = EEPROM_MAGIC;
2989 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2990 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2993 error = copyout(buf + e->offset, e->data, e->len);
2995 free(buf, M_DEVBUF);
2998 case CHELSIO_CLEAR_STATS: {
2999 if (!(sc->flags & FULL_INIT_DONE))
3003 t3_mac_update_stats(&pi->mac);
3004 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
3008 case CHELSIO_GET_UP_LA: {
3009 struct ch_up_la *la = (struct ch_up_la *)data;
3010 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
3014 if (la->bufsize < LA_BUFSIZE)
3018 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
3021 error = copyout(buf, la->data, la->bufsize);
3023 free(buf, M_DEVBUF);
3026 case CHELSIO_GET_UP_IOQS: {
3027 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3028 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3034 if (ioqs->bufsize < IOQS_BUFSIZE)
3038 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3041 v = (uint32_t *)buf;
3043 ioqs->ioq_rx_enable = *v++;
3044 ioqs->ioq_tx_enable = *v++;
3045 ioqs->ioq_rx_status = *v++;
3046 ioqs->ioq_tx_status = *v++;
3048 error = copyout(v, ioqs->data, ioqs->bufsize);
3051 free(buf, M_DEVBUF);
3054 case CHELSIO_SET_FILTER: {
3055 struct ch_filter *f = (struct ch_filter *)data;
3056 struct filter_info *p;
3057 unsigned int nfilters = sc->params.mc5.nfilters;
3059 if (!is_offload(sc))
3060 return (EOPNOTSUPP); /* No TCAM */
3061 if (!(sc->flags & FULL_INIT_DONE))
3062 return (EAGAIN); /* mc5 not setup yet */
3064 return (EBUSY); /* TOE will use TCAM */
3067 if (f->filter_id >= nfilters ||
3068 (f->val.dip && f->mask.dip != 0xffffffff) ||
3069 (f->val.sport && f->mask.sport != 0xffff) ||
3070 (f->val.dport && f->mask.dport != 0xffff) ||
3071 (f->val.vlan && f->mask.vlan != 0xfff) ||
3072 (f->val.vlan_prio &&
3073 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3074 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3075 f->qset >= SGE_QSETS ||
3076 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3079 /* Was allocated with M_WAITOK */
3080 KASSERT(sc->filters, ("filter table NULL\n"));
3082 p = &sc->filters[f->filter_id];
3086 bzero(p, sizeof(*p));
3087 p->sip = f->val.sip;
3088 p->sip_mask = f->mask.sip;
3089 p->dip = f->val.dip;
3090 p->sport = f->val.sport;
3091 p->dport = f->val.dport;
3092 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3093 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3095 p->mac_hit = f->mac_hit;
3096 p->mac_vld = f->mac_addr_idx != 0xffff;
3097 p->mac_idx = f->mac_addr_idx;
3098 p->pkt_type = f->proto;
3099 p->report_filter_id = f->want_filter_id;
3104 error = set_filter(sc, f->filter_id, p);
3109 case CHELSIO_DEL_FILTER: {
3110 struct ch_filter *f = (struct ch_filter *)data;
3111 struct filter_info *p;
3112 unsigned int nfilters = sc->params.mc5.nfilters;
3114 if (!is_offload(sc))
3115 return (EOPNOTSUPP);
3116 if (!(sc->flags & FULL_INIT_DONE))
3118 if (nfilters == 0 || sc->filters == NULL)
3120 if (f->filter_id >= nfilters)
3123 p = &sc->filters[f->filter_id];
3127 return (EFAULT); /* Read "Bad address" as "Bad index" */
3129 bzero(p, sizeof(*p));
3130 p->sip = p->sip_mask = 0xffffffff;
3132 p->vlan_prio = FILTER_NO_VLAN_PRI;
3134 error = set_filter(sc, f->filter_id, p);
3137 case CHELSIO_GET_FILTER: {
3138 struct ch_filter *f = (struct ch_filter *)data;
3139 struct filter_info *p;
3140 unsigned int i, nfilters = sc->params.mc5.nfilters;
3142 if (!is_offload(sc))
3143 return (EOPNOTSUPP);
3144 if (!(sc->flags & FULL_INIT_DONE))
3146 if (nfilters == 0 || sc->filters == NULL)
3149 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3150 for (; i < nfilters; i++) {
3151 p = &sc->filters[i];
3155 bzero(f, sizeof(*f));
3158 f->val.sip = p->sip;
3159 f->mask.sip = p->sip_mask;
3160 f->val.dip = p->dip;
3161 f->mask.dip = p->dip ? 0xffffffff : 0;
3162 f->val.sport = p->sport;
3163 f->mask.sport = p->sport ? 0xffff : 0;
3164 f->val.dport = p->dport;
3165 f->mask.dport = p->dport ? 0xffff : 0;
3166 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3167 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3168 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3170 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3171 0 : FILTER_NO_VLAN_PRI;
3172 f->mac_hit = p->mac_hit;
3173 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3174 f->proto = p->pkt_type;
3175 f->want_filter_id = p->report_filter_id;
3184 f->filter_id = 0xffffffff;
3188 return (EOPNOTSUPP);
3195 static __inline void
3196 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3199 uint32_t *p = (uint32_t *)(buf + start);
3201 for ( ; start <= end; start += sizeof(uint32_t))
3202 *p++ = t3_read_reg(ap, start);
3205 #define T3_REGMAP_SIZE (3 * 1024)
3207 cxgb_get_regs_len(void)
3209 return T3_REGMAP_SIZE;
3213 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3218 * bits 0..9: chip version
3219 * bits 10..15: chip revision
3220 * bit 31: set for PCIe cards
3222 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3225 * We skip the MAC statistics registers because they are clear-on-read.
3226 * Also reading multi-register stats would need to synchronize with the
3227 * periodic mac stats accumulation. Hard to justify the complexity.
3229 memset(buf, 0, cxgb_get_regs_len());
3230 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3231 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3232 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3233 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3234 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3235 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3236 XGM_REG(A_XGM_SERDES_STAT3, 1));
3237 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3238 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3242 alloc_filters(struct adapter *sc)
3244 struct filter_info *p;
3245 unsigned int nfilters = sc->params.mc5.nfilters;
3250 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3253 p = &sc->filters[nfilters - 1];
3255 p->vlan_prio = FILTER_NO_VLAN_PRI;
3256 p->pass = p->rss = p->valid = p->locked = 1;
3262 setup_hw_filters(struct adapter *sc)
3265 unsigned int nfilters = sc->params.mc5.nfilters;
3270 t3_enable_filters(sc);
3272 for (i = rc = 0; i < nfilters && !rc; i++) {
3273 if (sc->filters[i].locked)
3274 rc = set_filter(sc, i, &sc->filters[i]);
3281 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3285 struct ulp_txpkt *txpkt;
3286 struct work_request_hdr *wr;
3287 struct cpl_pass_open_req *oreq;
3288 struct cpl_set_tcb_field *sreq;
3290 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3291 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3293 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3294 sc->params.mc5.nfilters;
3296 m = m_gethdr(M_WAITOK, MT_DATA);
3297 m->m_len = m->m_pkthdr.len = len;
3298 bzero(mtod(m, char *), len);
3300 wr = mtod(m, struct work_request_hdr *);
3301 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3303 oreq = (struct cpl_pass_open_req *)(wr + 1);
3304 txpkt = (struct ulp_txpkt *)oreq;
3305 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3306 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3307 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3308 oreq->local_port = htons(f->dport);
3309 oreq->peer_port = htons(f->sport);
3310 oreq->local_ip = htonl(f->dip);
3311 oreq->peer_ip = htonl(f->sip);
3312 oreq->peer_netmask = htonl(f->sip_mask);
3314 oreq->opt0l = htonl(F_NO_OFFLOAD);
3315 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3316 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3317 V_VLAN_PRI(f->vlan_prio >> 1) |
3318 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3319 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3320 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3322 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3323 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3324 (f->report_filter_id << 15) | (1 << 23) |
3325 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3326 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3329 if (f->pass && !f->rss) {
3330 len = sizeof(*sreq);
3331 m = m_gethdr(M_WAITOK, MT_DATA);
3332 m->m_len = m->m_pkthdr.len = len;
3333 bzero(mtod(m, char *), len);
3334 sreq = mtod(m, struct cpl_set_tcb_field *);
3335 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3336 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3337 (u64)sc->rrss_map[f->qset] << 19);
3344 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3345 unsigned int word, u64 mask, u64 val)
3347 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3348 req->reply = V_NO_REPLY(1);
3350 req->word = htons(word);
3351 req->mask = htobe64(mask);
3352 req->val = htobe64(val);
3356 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3357 unsigned int word, u64 mask, u64 val)
3359 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3361 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3362 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3363 mk_set_tcb_field(req, tid, word, mask, val);