1 /**************************************************************************
3 Copyright (c) 2007-2008, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
32 ***************************************************************************/
35 #ifndef _CXGB_ADAPTER_H_
36 #define _CXGB_ADAPTER_H_
39 #include <sys/mutex.h>
43 #include <sys/socket.h>
44 #include <sys/sockio.h>
45 #include <sys/condvar.h>
47 #include <net/ethernet.h>
49 #include <net/if_media.h>
50 #include <net/if_dl.h>
51 #include <netinet/tcp_lro.h>
53 #include <machine/bus.h>
54 #include <machine/resource.h>
56 #include <sys/bus_dma.h>
57 #include <dev/pci/pcireg.h>
58 #include <dev/pci/pcivar.h>
62 #include <cxgb_osdep.h>
64 #include <ulp/toecore/cxgb_toedev.h>
65 #include <sys/mbufq.h>
67 #include <dev/cxgb/cxgb_osdep.h>
68 #include <dev/cxgb/t3cdev.h>
69 #include <dev/cxgb/sys/mbufq.h>
70 #include <dev/cxgb/ulp/toecore/cxgb_toedev.h>
77 extern int cxgb_debug;
80 #define MTX_INIT(lock, lockname, class, flags) \
82 printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \
83 mtx_init((lock), lockname, class, flags); \
86 #define MTX_DESTROY(lock) \
88 printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \
89 mtx_destroy((lock)); \
92 #define SX_INIT(lock, lockname) \
94 printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \
95 sx_init((lock), lockname); \
98 #define SX_DESTROY(lock) \
100 printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \
101 sx_destroy((lock)); \
104 #define MTX_INIT mtx_init
105 #define MTX_DESTROY mtx_destroy
106 #define SX_INIT sx_init
107 #define SX_DESTROY sx_destroy
111 struct adapter *adapter;
114 const struct port_type_info *port_type;
117 struct link_config link_config;
118 struct ifmedia media;
130 uint8_t hw_addr[ETHER_ADDR_LEN];
131 struct task timer_reclaim_task;
132 struct cdev *port_cdev;
134 #define PORT_LOCK_NAME_LEN 32
135 #define PORT_NAME_LEN 32
136 char lockbuf[PORT_LOCK_NAME_LEN];
137 char namebuf[PORT_NAME_LEN];
140 enum { /* adapter flags */
141 FULL_INIT_DONE = (1 << 0),
142 USING_MSI = (1 << 1),
143 USING_MSIX = (1 << 2),
144 QUEUES_BOUND = (1 << 3),
145 FW_UPTODATE = (1 << 4),
146 TPS_UPTODATE = (1 << 5),
147 CXGB_SHUTDOWN = (1 << 6),
148 CXGB_OFLD_INIT = (1 << 7),
149 TP_PARITY_INIT = (1 << 8),
152 #define FL_Q_SIZE 4096
153 #define JUMBO_Q_SIZE 1024
154 #define RSPQ_Q_SIZE 1024
155 #define TX_ETH_Q_SIZE 1024
163 * work request size in bytes
165 #define WR_LEN (WR_FLITS * 8)
166 #define PIO_LEN (WR_LEN - sizeof(struct cpl_tx_pkt_lso))
169 unsigned short enabled;
170 struct lro_ctrl ctrl;
173 #define RX_BUNDLE_SIZE 8
183 uint32_t holdoff_tmr;
184 uint32_t next_holdoff;
186 uint32_t async_notif;
188 uint32_t offload_pkts;
189 uint32_t offload_bundles;
191 uint32_t unhandled_irqs;
193 bus_addr_t phys_addr;
194 bus_dma_tag_t desc_tag;
195 bus_dmamap_t desc_map;
197 struct t3_mbuf_hdr rspq_mh;
198 struct rsp_desc *desc;
200 #define RSPQ_NAME_LEN 32
201 char lockbuf[RSPQ_NAME_LEN];
202 uint32_t rspq_dump_start;
203 uint32_t rspq_dump_count;
206 #ifndef DISABLE_MBUF_IOVEC
207 #define rspq_mbuf rspq_mh.mh_head
220 bus_addr_t phys_addr;
223 bus_dma_tag_t desc_tag;
224 bus_dmamap_t desc_map;
225 bus_dma_tag_t entry_tag;
227 struct rx_desc *desc;
228 struct rx_sw_desc *sdesc;
235 #define TXQ_TRANSMITTING 0x1
248 struct tx_desc *desc;
249 struct tx_sw_desc *sdesc;
251 bus_addr_t phys_addr;
252 struct task qresume_task;
253 struct task qreclaim_task;
254 struct port_info *port;
258 bus_dma_tag_t desc_tag;
259 bus_dmamap_t desc_map;
260 bus_dma_tag_t entry_tag;
261 struct mbuf_head sendq;
263 * cleanq should really be an buf_ring to avoid extra
266 struct mbuf_head cleanq;
267 struct buf_ring txq_mr;
270 uint32_t txq_skipped;
271 uint32_t txq_coalesced;
272 uint32_t txq_enqueued;
273 uint32_t txq_dump_start;
274 uint32_t txq_dump_count;
275 unsigned long txq_frees;
277 struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1];
278 #define TXQ_NAME_LEN 32
279 char lockbuf[TXQ_NAME_LEN];
284 SGE_PSTAT_TSO, /* # of TSO requests */
285 SGE_PSTAT_RX_CSUM_GOOD, /* # of successful RX csum offloads */
286 SGE_PSTAT_TX_CSUM, /* # of TX checksum offloads */
287 SGE_PSTAT_VLANEX, /* # of VLAN tag extractions */
288 SGE_PSTAT_VLANINS, /* # of VLAN tag insertions */
291 #define SGE_PSTAT_MAX (SGE_PSTAT_VLANINS+1)
293 #define QS_EXITING 0x1
294 #define QS_RUNNING 0x2
298 struct sge_rspq rspq;
299 struct sge_fl fl[SGE_RXQ_PER_SET];
300 struct lro_state lro;
301 struct sge_txq txq[SGE_TXQ_PER_SET];
302 uint32_t txq_stopped; /* which Tx queues are stopped */
303 uint64_t port_stats[SGE_PSTAT_MAX];
304 struct port_info *port;
305 int idx; /* qset # */
310 #define QS_NAME_LEN 32
311 char namebuf[QS_NAME_LEN];
315 struct sge_qset qs[SGE_QSETS];
324 TAILQ_ENTRY(adapter) adapter_entry;
326 /* PCI register resources */
328 struct resource *regs_res;
330 struct resource *udbs_res;
331 bus_space_handle_t bh;
337 bus_dma_tag_t parent_dmat;
338 bus_dma_tag_t rx_dmat;
339 bus_dma_tag_t rx_jumbo_dmat;
340 bus_dma_tag_t tx_dmat;
342 /* Interrupt resources */
343 struct resource *irq_res;
347 uint32_t msix_regs_rid;
348 struct resource *msix_regs_res;
350 struct resource *msix_irq_res[SGE_QSETS];
351 int msix_irq_rid[SGE_QSETS];
352 void *msix_intr_tag[SGE_QSETS];
353 uint8_t rxpkt_map[8]; /* maps RX_PKT interface values to port ids */
354 uint8_t rrss_map[SGE_QSETS]; /* revers RSS map table */
355 uint16_t rspq_map[RSS_TABLE_SIZE]; /* maps 7-bit cookie to qidx */
357 uint8_t fill[SGE_QSETS];
361 #define tunq_fill u.fill
362 #define tunq_coalesce u.coalesce
364 struct filter_info *filters;
367 struct task ext_intr_task;
368 struct task slow_intr_task;
369 struct task tick_task;
370 struct taskqueue *tq;
371 struct callout cxgb_tick_ch;
372 struct callout sge_timer_ch;
374 unsigned int check_task_cnt;
376 /* Register lock for use by the hardware layer */
377 struct mtx mdio_lock;
378 struct mtx elmer_lock;
380 /* Bookkeeping for the hardware layer */
381 struct adapter_params params;
382 unsigned int slow_intr_mask;
383 unsigned long irq_stats[IRQ_NUM_STATS];
391 struct port_info port[MAX_NPORTS];
392 device_t portdev[MAX_NPORTS];
395 uint32_t open_device_map;
396 uint32_t registered_device_map;
402 driver_intr_t *cxgb_intr;
405 #define ADAPTER_LOCK_NAME_LEN 32
406 char lockbuf[ADAPTER_LOCK_NAME_LEN];
407 char reglockbuf[ADAPTER_LOCK_NAME_LEN];
408 char mdiolockbuf[ADAPTER_LOCK_NAME_LEN];
409 char elmerlockbuf[ADAPTER_LOCK_NAME_LEN];
415 struct port_info *port;
419 #define MDIO_LOCK(adapter) mtx_lock(&(adapter)->mdio_lock)
420 #define MDIO_UNLOCK(adapter) mtx_unlock(&(adapter)->mdio_lock)
421 #define ELMR_LOCK(adapter) mtx_lock(&(adapter)->elmer_lock)
422 #define ELMR_UNLOCK(adapter) mtx_unlock(&(adapter)->elmer_lock)
426 #define PORT_LOCK(port) sx_xlock(&(port)->lock);
427 #define PORT_UNLOCK(port) sx_xunlock(&(port)->lock);
428 #define PORT_LOCK_INIT(port, name) SX_INIT(&(port)->lock, name)
429 #define PORT_LOCK_DEINIT(port) SX_DESTROY(&(port)->lock)
430 #define PORT_LOCK_ASSERT_OWNED(port) sx_assert(&(port)->lock, SA_LOCKED)
432 #define ADAPTER_LOCK(adap) sx_xlock(&(adap)->lock);
433 #define ADAPTER_UNLOCK(adap) sx_xunlock(&(adap)->lock);
434 #define ADAPTER_LOCK_INIT(adap, name) SX_INIT(&(adap)->lock, name)
435 #define ADAPTER_LOCK_DEINIT(adap) SX_DESTROY(&(adap)->lock)
436 #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) sx_assert(&(adap)->lock, SA_UNLOCKED)
438 #define PORT_LOCK(port) mtx_lock(&(port)->lock);
439 #define PORT_UNLOCK(port) mtx_unlock(&(port)->lock);
440 #define PORT_LOCK_INIT(port, name) mtx_init(&(port)->lock, name, 0, MTX_DEF)
441 #define PORT_LOCK_DEINIT(port) mtx_destroy(&(port)->lock)
442 #define PORT_LOCK_ASSERT_OWNED(port) mtx_assert(&(port)->lock, MA_OWNED)
444 #define ADAPTER_LOCK(adap) mtx_lock(&(adap)->lock);
445 #define ADAPTER_UNLOCK(adap) mtx_unlock(&(adap)->lock);
446 #define ADAPTER_LOCK_INIT(adap, name) mtx_init(&(adap)->lock, name, 0, MTX_DEF)
447 #define ADAPTER_LOCK_DEINIT(adap) mtx_destroy(&(adap)->lock)
448 #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) mtx_assert(&(adap)->lock, MO_NOTOWNED)
452 static __inline uint32_t
453 t3_read_reg(adapter_t *adapter, uint32_t reg_addr)
455 return (bus_space_read_4(adapter->bt, adapter->bh, reg_addr));
459 t3_write_reg(adapter_t *adapter, uint32_t reg_addr, uint32_t val)
461 bus_space_write_4(adapter->bt, adapter->bh, reg_addr, val);
465 t3_os_pci_read_config_4(adapter_t *adapter, int reg, uint32_t *val)
467 *val = pci_read_config(adapter->dev, reg, 4);
471 t3_os_pci_write_config_4(adapter_t *adapter, int reg, uint32_t val)
473 pci_write_config(adapter->dev, reg, val, 4);
477 t3_os_pci_read_config_2(adapter_t *adapter, int reg, uint16_t *val)
479 *val = pci_read_config(adapter->dev, reg, 2);
483 t3_os_pci_write_config_2(adapter_t *adapter, int reg, uint16_t val)
485 pci_write_config(adapter->dev, reg, val, 2);
488 static __inline uint8_t *
489 t3_get_next_mcaddr(struct t3_rx_mode *rm)
491 uint8_t *macaddr = NULL;
492 struct ifnet *ifp = rm->port->ifp;
493 struct ifmultiaddr *ifma;
497 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
498 if (ifma->ifma_addr->sa_family != AF_LINK)
501 macaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
514 t3_init_rx_mode(struct t3_rx_mode *rm, struct port_info *port)
520 static __inline struct port_info *
521 adap2pinfo(struct adapter *adap, int idx)
523 return &adap->port[idx];
526 int t3_os_find_pci_capability(adapter_t *adapter, int cap);
527 int t3_os_pci_save_state(struct adapter *adapter);
528 int t3_os_pci_restore_state(struct adapter *adapter);
529 void t3_os_link_changed(adapter_t *adapter, int port_id, int link_status,
530 int speed, int duplex, int fc);
531 void t3_os_phymod_changed(struct adapter *adap, int port_id);
532 void t3_sge_err_intr_handler(adapter_t *adapter);
533 int t3_offload_tx(struct t3cdev *, struct mbuf *);
534 void t3_os_ext_intr_handler(adapter_t *adapter);
535 void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]);
536 int t3_mgmt_tx(adapter_t *adap, struct mbuf *m);
539 int t3_sge_alloc(struct adapter *);
540 int t3_sge_free(struct adapter *);
541 int t3_sge_alloc_qset(adapter_t *, uint32_t, int, int, const struct qset_params *,
542 int, struct port_info *);
543 void t3_free_sge_resources(adapter_t *);
544 void t3_sge_start(adapter_t *);
545 void t3_sge_stop(adapter_t *);
546 void t3b_intr(void *data);
547 void t3_intr_msi(void *data);
548 void t3_intr_msix(void *data);
549 int t3_encap(struct sge_qset *, struct mbuf **, int);
551 int t3_sge_init_adapter(adapter_t *);
552 int t3_sge_reset_adapter(adapter_t *);
553 int t3_sge_init_port(struct port_info *);
554 void t3_sge_deinit_sw(adapter_t *);
555 void t3_free_tx_desc(struct sge_txq *q, int n);
556 void t3_free_tx_desc_all(struct sge_txq *q);
558 void t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad);
560 void t3_add_attach_sysctls(adapter_t *sc);
561 void t3_add_configured_sysctls(adapter_t *sc);
562 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
563 unsigned char *data);
564 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p);
566 #define CXGB_TICKS(a) ((a)->params.linkpoll_period ? \
567 (hz * (a)->params.linkpoll_period) / 10 : \
568 (a)->params.stats_update_period * hz)
571 * XXX figure out how we can return this to being private to sge
573 #define desc_reclaimable(q) ((int)((q)->processed - (q)->cleaned - TX_MAX_DESC))
575 #define container_of(p, stype, field) ((stype *)(((uint8_t *)(p)) - offsetof(stype, field)))
577 static __inline struct sge_qset *
578 fl_to_qset(struct sge_fl *q, int qidx)
580 return container_of(q, struct sge_qset, fl[qidx]);
583 static __inline struct sge_qset *
584 rspq_to_qset(struct sge_rspq *q)
586 return container_of(q, struct sge_qset, rspq);
589 static __inline struct sge_qset *
590 txq_to_qset(struct sge_txq *q, int qidx)
592 return container_of(q, struct sge_qset, txq[qidx]);
595 static __inline struct adapter *
596 tdev2adap(struct t3cdev *d)
598 return container_of(d, struct adapter, tdev);
603 #define OFFLOAD_DEVMAP_BIT 15
604 static inline int offload_running(adapter_t *adapter)
606 return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
609 int cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m);
610 int cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *m);
611 void cxgb_pcpu_shutdown_threads(struct adapter *sc);
612 void cxgb_pcpu_startup_threads(struct adapter *sc);
614 int process_responses(adapter_t *adap, struct sge_qset *qs, int budget);
615 void t3_free_qset(adapter_t *sc, struct sge_qset *q);
616 void cxgb_start(struct ifnet *ifp);
617 void refill_fl_service(adapter_t *adap, struct sge_fl *fl);