2 * Copyright (c) 2012 Chelsio Communications, Inc.
5 * Chelsio T5xx iSCSI driver
6 * cxgbei_ulp2_ddp.c: Chelsio iSCSI DDP Manager.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
34 #include "opt_inet6.h"
37 #include <sys/types.h>
38 #include <sys/module.h>
39 #include <sys/systm.h>
40 #include <sys/errno.h>
41 #include <sys/param.h>
42 #include <sys/kernel.h>
43 #include <sys/socket.h>
44 #include <sys/socketvar.h>
47 #include <sys/mutex.h>
48 #include <sys/condvar.h>
50 #include <netinet/in.h>
51 #include <netinet/in_pcb.h>
52 #include <netinet/toecore.h>
53 #include <netinet/tcp_var.h>
54 #include <netinet/tcp_fsm.h>
56 #include <dev/iscsi/icl.h>
57 #include <dev/iscsi/iscsi_proto.h>
59 #include "common/common.h"
60 #include "common/t4_msg.h"
61 #include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */
62 #include "tom/t4_tom.h"
64 #include "cxgbei_ulp2_ddp.h"
67 * Map a single buffer address.
70 ulp2_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
76 KASSERT(nseg == 1, ("%s: %d segments returned!", __func__, nseg));
82 * iSCSI Direct Data Placement
84 * T4/5 ulp2 h/w can directly place the iSCSI Data-In or Data-Out PDU's
85 * payload into pre-posted final destination host-memory buffers based on the
86 * Initiator Task Tag (ITT) in Data-In or Target Task Tag (TTT) in Data-Out
89 * The host memory address is programmed into h/w in the format of pagepod
91 * The location of the pagepod entry is encoded into ddp tag which is used or
92 * is the base for ITT/TTT.
97 ddp_find_unused_entries(struct cxgbei_data *ci, u_int start, u_int max,
98 u_int count, u_int *idx, struct cxgbei_ulp2_gather_list *gl)
100 unsigned int i, j, k;
102 /* not enough entries */
103 if (max - start < count)
107 mtx_lock(&ci->map_lock);
108 for (i = start; i < max;) {
109 for (j = 0, k = i; j < count; j++, k++) {
114 for (j = 0, k = i; j < count; j++, k++)
116 mtx_unlock(&ci->map_lock);
122 mtx_unlock(&ci->map_lock);
127 ddp_unmark_entries(struct cxgbei_data *ci, u_int start, u_int count)
130 mtx_lock(&ci->map_lock);
131 memset(&ci->gl_map[start], 0,
132 count * sizeof(struct cxgbei_ulp2_gather_list *));
133 mtx_unlock(&ci->map_lock);
137 ddp_gl_unmap(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl)
144 for (i = 0; i < gl->nelem; i++) {
145 bus_dmamap_unload(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map);
146 bus_dmamap_destroy(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map);
151 ddp_gl_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl)
158 mtx_lock(&ci->map_lock);
159 for (i = 0; i < gl->nelem; i++) {
160 rc = bus_dmamap_create(ci->ulp_ddp_tag, 0,
161 &gl->dma_sg[i].bus_map);
164 rc = bus_dmamap_load(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map,
165 gl->pages[i], PAGE_SIZE, ulp2_dma_map_addr,
166 &pa, BUS_DMA_NOWAIT);
169 gl->dma_sg[i].phys_addr = pa;
171 mtx_unlock(&ci->map_lock);
177 u_int nelem = gl->nelem;
180 ddp_gl_unmap(ci, gl);
187 * cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec - build ddp page buffer list
188 * @xferlen: total buffer length
189 * @sgl: page buffer scatter-gather list (struct cxgbei_sgl)
190 * @sgcnt: # of page buffers
191 * @gfp: allocation mode
193 * construct a ddp page buffer list from the scsi scattergather list.
194 * coalesce buffers as much as possible, and obtain dma addresses for
197 * Return the cxgbei_ulp2_gather_list constructed from the page buffers if the
198 * memory can be used for ddp. Return NULL otherwise.
200 struct cxgbei_ulp2_gather_list *
201 cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int xferlen, struct cxgbei_sgl *sgl,
202 u_int sgcnt, struct cxgbei_data *ci, int gfp)
204 struct cxgbei_ulp2_gather_list *gl;
205 struct cxgbei_sgl *sg = sgl;
206 void *sgpage = (void *)((u64)sg->sg_addr & (~PAGE_MASK));
207 unsigned int sglen = sg->sg_length;
208 unsigned int sgoffset = (u64)sg->sg_addr & PAGE_MASK;
209 unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
213 if (xferlen <= DDP_THRESHOLD) {
214 CTR2(KTR_CXGBE, "xfer %u < threshold %u, no ddp.",
215 xferlen, DDP_THRESHOLD);
219 gl = malloc(sizeof(struct cxgbei_ulp2_gather_list) +
220 npages * (sizeof(struct dma_segments) + sizeof(void *)),
221 M_DEVBUF, M_NOWAIT | M_ZERO);
225 gl->pages = (void **)&gl->dma_sg[npages];
226 gl->length = xferlen;
227 gl->offset = sgoffset;
228 gl->pages[0] = sgpage;
230 "%s: xferlen:0x%x len:0x%x off:0x%x sg_addr:%p npages:%d",
231 __func__, xferlen, gl->length, gl->offset, sg->sg_addr, npages);
233 for (i = 1, sg = sg_next(sg); i < sgcnt; i++, sg = sg_next(sg)) {
234 void *page = sg->sg_addr;
236 if (sgpage == page && sg->sg_offset == sgoffset + sglen)
237 sglen += sg->sg_length;
239 /* make sure the sgl is fit for ddp:
240 * each has the same page size, and
241 * all of the middle pages are used completely
243 if ((j && sgoffset) ||
245 ((sglen + sgoffset) & ~CXGBEI_PAGE_MASK))){
250 if (j == gl->nelem || sg->sg_offset) {
254 sglen = sg->sg_length;
255 sgoffset = sg->sg_offset;
261 if (ddp_gl_map(ci, gl) < 0)
272 * cxgbei_ulp2_ddp_release_gl - release a page buffer list
273 * @gl: a ddp page buffer list
274 * @pdev: pci_dev used for pci_unmap
275 * free a ddp page buffer list resulted from cxgbei_ulp2_ddp_make_gl().
278 cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *ci,
279 struct cxgbei_ulp2_gather_list *gl)
282 ddp_gl_unmap(ci, gl);
287 * cxgbei_ulp2_ddp_tag_reserve - set up ddp for a data transfer
288 * @ci: adapter's ddp info
289 * @tid: connection id
290 * @tformat: tag format
291 * @tagp: contains s/w tag initially, will be updated with ddp/hw tag
292 * @gl: the page momory list
293 * @gfp: allocation mode
295 * ddp setup for a given page buffer list and construct the ddp tag.
296 * return 0 if success, < 0 otherwise.
299 cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *ci, void *icc, u_int tid,
300 struct cxgbei_ulp2_tag_format *tformat, u32 *tagp,
301 struct cxgbei_ulp2_gather_list *gl, int gfp, int reply)
303 struct cxgbei_ulp2_pagepod_hdr hdr;
311 if (!gl || !gl->nelem || gl->length < DDP_THRESHOLD)
314 npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT;
316 if (ci->idx_last == ci->nppods)
317 rc = ddp_find_unused_entries(ci, 0, ci->nppods, npods, &idx,
320 rc = ddp_find_unused_entries(ci, ci->idx_last + 1,
321 ci->nppods, npods, &idx, gl);
322 if (rc && ci->idx_last >= npods) {
323 rc = ddp_find_unused_entries(ci, 0,
324 min(ci->idx_last + npods, ci->nppods),
329 CTR3(KTR_CXGBE, "xferlen %u, gl %u, npods %u NO DDP.",
330 gl->length, gl->nelem, npods);
334 tag = cxgbei_ulp2_ddp_tag_base(idx, ci->colors, tformat, sw_tag);
335 CTR4(KTR_CXGBE, "%s: sw_tag:0x%x idx:0x%x tag:0x%x",
336 __func__, sw_tag, idx, tag);
339 hdr.vld_tid = htonl(F_IPPOD_VALID | V_IPPOD_TID(tid));
340 hdr.pgsz_tag_clr = htonl(tag & ci->rsvd_tag_mask);
341 hdr.maxoffset = htonl(gl->length);
342 hdr.pgoffset = htonl(gl->offset);
344 rc = t4_ddp_set_map(ci, icc, &hdr, idx, npods, gl, reply);
353 ddp_unmark_entries(ci, idx, npods);
358 * cxgbei_ulp2_ddp_tag_release - release a ddp tag
359 * @ci: adapter's ddp info
361 * ddp cleanup for a given ddp tag and release all the resources held
364 cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *ci, uint32_t tag,
365 struct icl_cxgbei_conn *icc)
372 idx = (tag >> IPPOD_IDX_SHIFT) & ci->idx_mask;
373 CTR3(KTR_CXGBE, "tag:0x%x idx:0x%x nppods:0x%x",
374 tag, idx, ci->nppods);
375 if (idx < ci->nppods) {
376 struct cxgbei_ulp2_gather_list *gl = ci->gl_map[idx];
379 if (!gl || !gl->nelem) {
381 "release 0x%x, idx 0x%x, gl 0x%p, %u.",
382 tag, idx, gl, gl ? gl->nelem : 0);
385 npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT;
386 CTR3(KTR_CXGBE, "ddp tag 0x%x, release idx 0x%x, npods %u.",
388 t4_ddp_clear_map(ci, gl, tag, idx, npods, icc);
389 ddp_unmark_entries(ci, idx, npods);
390 cxgbei_ulp2_ddp_release_gl(ci, gl);
392 CTR3(KTR_CXGBE, "ddp tag 0x%x, idx 0x%x > max 0x%x.",
393 tag, idx, ci->nppods);
397 * cxgbei_ddp_cleanup - release the adapter's ddp resources
400 cxgbei_ddp_cleanup(struct cxgbei_data *ci)
404 while (i < ci->nppods) {
405 struct cxgbei_ulp2_gather_list *gl = ci->gl_map[i];
407 int npods = (gl->nelem + IPPOD_PAGES_MAX - 1)
408 >> IPPOD_PAGES_SHIFT;
414 free(ci->colors, M_CXGBE);
415 free(ci->gl_map, M_CXGBE);