1 /**************************************************************************
3 Copyright (c) 2007, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
31 ***************************************************************************/
36 /* Should be 1 or 2 indicating single or double kernel buffers. */
37 #define NUM_DDP_KBUF 2
39 /* min receive window for a connection to be considered for DDP */
40 #define MIN_DDP_RCV_WIN (48 << 10)
42 /* amount of Rx window not available to DDP to avoid window exhaustion */
43 #define DDP_RSVD_WIN (16 << 10)
45 /* # of sentinel invalid page pods at the end of a group of valid page pods */
46 #define NUM_SENTINEL_PPODS 0
48 /* # of pages a pagepod can hold without needing another pagepod */
51 /* page pods are allocated in groups of this size (must be power of 2) */
52 #define PPOD_CLUSTER_SIZE 16
54 /* for each TID we reserve this many page pods up front */
55 #define RSVD_PPODS_PER_TID 1
59 uint32_t pp_pgsz_tag_color;
60 uint32_t pp_max_offset;
61 uint32_t pp_page_offset;
66 #define PPOD_SIZE sizeof(struct pagepod)
69 #define M_PPOD_TID 0xFFFFFF
70 #define V_PPOD_TID(x) ((x) << S_PPOD_TID)
72 #define S_PPOD_VALID 24
73 #define V_PPOD_VALID(x) ((x) << S_PPOD_VALID)
74 #define F_PPOD_VALID V_PPOD_VALID(1U)
76 #define S_PPOD_COLOR 0
77 #define M_PPOD_COLOR 0x3F
78 #define V_PPOD_COLOR(x) ((x) << S_PPOD_COLOR)
81 #define M_PPOD_TAG 0xFFFFFF
82 #define V_PPOD_TAG(x) ((x) << S_PPOD_TAG)
84 #define S_PPOD_PGSZ 30
85 #define M_PPOD_PGSZ 0x3
86 #define V_PPOD_PGSZ(x) ((x) << S_PPOD_PGSZ)
89 #include <vm/vm_page.h>
90 #include <machine/bus.h>
92 /* DDP gather lists can specify an offset only for the first page. */
93 struct ddp_gather_list {
94 unsigned int dgl_length;
95 unsigned int dgl_offset;
96 unsigned int dgl_nelem;
97 vm_page_t dgl_pages[0];
100 struct ddp_buf_state {
101 unsigned int cur_offset; /* offset of latest DDP notification */
103 struct ddp_gather_list *gl;
107 struct ddp_buf_state buf_state[2]; /* per buffer state */
109 unsigned short kbuf_noinval;
110 unsigned short kbuf_idx; /* which HW buffer is used for kbuf */
111 struct ddp_gather_list *ubuf;
112 int user_ddp_pending;
113 unsigned int ubuf_nppods; /* # of page pods for buffer 1 */
114 unsigned int ubuf_tag;
115 unsigned int ubuf_ddp_ready;
118 unsigned int kbuf_posted;
119 unsigned int kbuf_nppods[NUM_DDP_KBUF];
120 unsigned int kbuf_tag[NUM_DDP_KBUF];
121 struct ddp_gather_list *kbuf[NUM_DDP_KBUF]; /* kernel buffer for DDP prefetch */
124 /* buf_state flags */
126 DDP_BF_NOINVAL = 1 << 0, /* buffer is set to NO_INVALIDATE */
127 DDP_BF_NOCOPY = 1 << 1, /* DDP to final dest, no copy needed */
128 DDP_BF_NOFLIP = 1 << 2, /* buffer flips after GET_TCB_RPL */
129 DDP_BF_PSH = 1 << 3, /* set in skb->flags if the a DDP was
130 completed with a segment having the
132 DDP_BF_NODATA = 1 << 4, /* buffer completed before filling */
135 #include <ulp/tom/cxgb_toepcb.h>
139 * Returns 1 if a UBUF DMA buffer might be active.
142 t3_ddp_ubuf_pending(struct toepcb *toep)
144 struct ddp_state *p = &toep->tp_ddp_state;
146 /* When the TOM_TUNABLE(ddp) is enabled, we're always in ULP_MODE DDP,
147 * but DDP_STATE() is only valid if the connection actually enabled
150 if (p->kbuf[0] == NULL)
153 return (p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY)) ||
154 (p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY));
157 int t3_setup_ppods(struct toepcb *toep, const struct ddp_gather_list *gl,
158 unsigned int nppods, unsigned int tag, unsigned int maxoff,
159 unsigned int pg_off, unsigned int color);
160 int t3_alloc_ppods(struct tom_data *td, unsigned int n, int *tag);
161 void t3_free_ppods(struct tom_data *td, unsigned int tag, unsigned int n);
162 void t3_free_ddp_gl(struct ddp_gather_list *gl);
163 int t3_ddp_copy(const struct mbuf *m, int offset, struct uio *uio, int len);
164 //void t3_repost_kbuf(struct socket *so, int modulate, int activate);
165 void t3_post_kbuf(struct toepcb *toep, int modulate, int nonblock);
166 int t3_post_ubuf(struct toepcb *toep, const struct uio *uio, int nonblock,
167 int rcv_flags, int modulate, int post_kbuf);
168 void t3_cancel_ubuf(struct toepcb *toep, struct sockbuf *rcv);
169 int t3_overlay_ubuf(struct toepcb *toep, struct sockbuf *rcv,
170 const struct uio *uio, int nonblock,
171 int rcv_flags, int modulate, int post_kbuf);
172 int t3_enter_ddp(struct toepcb *toep, unsigned int kbuf_size, unsigned int waitall, int nonblock);
173 void t3_cleanup_ddp(struct toepcb *toep);
174 void t3_release_ddp_resources(struct toepcb *toep);
175 void t3_cancel_ddpbuf(struct toepcb *, unsigned int bufidx);
176 void t3_overlay_ddpbuf(struct toepcb *, unsigned int bufidx, unsigned int tag0,
177 unsigned int tag1, unsigned int len);
178 void t3_setup_ddpbufs(struct toepcb *, unsigned int len0, unsigned int offset0,
179 unsigned int len1, unsigned int offset1,
180 uint64_t ddp_flags, uint64_t flag_mask, int modulate);
181 #endif /* T3_DDP_H */