2 /**************************************************************************
4 Copyright (c) 2007, Chelsio Inc.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
29 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
40 #include <sys/pciio.h>
42 #include <machine/bus.h>
43 #include <machine/resource.h>
44 #include <sys/bus_dma.h>
46 #include <sys/ioccom.h>
48 #include <sys/mutex.h>
49 #include <sys/rwlock.h>
50 #include <sys/linker.h>
51 #include <sys/firmware.h>
52 #include <sys/socket.h>
53 #include <sys/sockio.h>
55 #include <sys/sysctl.h>
56 #include <sys/syslog.h>
57 #include <sys/queue.h>
58 #include <sys/taskqueue.h>
60 #include <sys/queue.h>
61 #include <sys/libkern.h>
63 #include <netinet/in.h>
64 #include <rdma/ib_verbs.h>
65 #include <rdma/ib_umem.h>
66 #include <rdma/ib_user_verbs.h>
67 #include <linux/idr.h>
68 #include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
70 #include <cxgb_include.h>
71 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
72 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
73 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
74 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
75 #include <ulp/iw_cxgb/iw_cxgb.h>
76 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
77 #include <ulp/iw_cxgb/iw_cxgb_user.h>
80 * Get one cq entry from cxio and map it to openib.
85 * -EAGAIN caller must try again
86 * any other neg errno fatal error
88 static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
91 struct iwch_qp *qhp = NULL;
92 struct t3_cqe cqe, *rd_cqe;
99 rd_cqe = cxio_next_cqe(&chp->cq);
104 qhp = get_qhp(rhp, CQE_QPID(*rd_cqe));
108 mtx_lock(&qhp->lock);
111 ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie,
113 if (t3a_device(chp->rhp) && credit) {
114 CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__,
115 credit, chp->cq.cqid);
116 cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit);
127 wc->vendor_err = CQE_STATUS(cqe);
129 CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x",
130 CQE_QPID(cqe), CQE_TYPE(cqe),
131 CQE_OPCODE(cqe), CQE_STATUS(cqe));
132 CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx",
133 CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie);
135 if (CQE_TYPE(cqe) == 0) {
136 if (!CQE_STATUS(cqe))
137 wc->byte_len = CQE_LEN(cqe);
140 wc->opcode = IB_WC_RECV;
142 switch (CQE_OPCODE(cqe)) {
144 wc->opcode = IB_WC_RDMA_WRITE;
147 wc->opcode = IB_WC_RDMA_READ;
148 wc->byte_len = CQE_LEN(cqe);
151 case T3_SEND_WITH_SE:
152 wc->opcode = IB_WC_SEND;
155 wc->opcode = IB_WC_BIND_MW;
158 /* these aren't supported yet */
159 case T3_SEND_WITH_INV:
160 case T3_SEND_WITH_SE_INV:
162 case T3_FAST_REGISTER:
164 log(LOG_ERR, "Unexpected opcode %d "
165 "in the CQE received for QPID=0x%0x\n",
166 CQE_OPCODE(cqe), CQE_QPID(cqe));
173 wc->status = IB_WC_WR_FLUSH_ERR;
176 switch (CQE_STATUS(cqe)) {
177 case TPT_ERR_SUCCESS:
178 wc->status = IB_WC_SUCCESS;
181 wc->status = IB_WC_LOC_ACCESS_ERR;
184 wc->status = IB_WC_LOC_PROT_ERR;
188 wc->status = IB_WC_LOC_ACCESS_ERR;
191 wc->status = IB_WC_GENERAL_ERR;
194 wc->status = IB_WC_LOC_LEN_ERR;
196 case TPT_ERR_INVALIDATE_SHARED_MR:
197 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
198 wc->status = IB_WC_MW_BIND_ERR;
202 case TPT_ERR_PDU_LEN_ERR:
203 case TPT_ERR_OUT_OF_RQE:
204 case TPT_ERR_DDP_VERSION:
205 case TPT_ERR_RDMA_VERSION:
206 case TPT_ERR_DDP_QUEUE_NUM:
210 case TPT_ERR_MSN_RANGE:
211 case TPT_ERR_IRD_OVERFLOW:
213 wc->status = IB_WC_FATAL_ERR;
215 case TPT_ERR_SWFLUSH:
216 wc->status = IB_WC_WR_FLUSH_ERR;
219 log(LOG_ERR, "Unexpected cqe_status 0x%x for "
220 "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe));
226 mtx_unlock(&qhp->lock);
230 int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
232 struct iwch_dev *rhp;
237 chp = to_iwch_cq(ibcq);
240 mtx_lock(&chp->lock);
241 for (npolled = 0; npolled < num_entries; ++npolled) {
247 * Because T3 can post CQEs that are _not_ associated
248 * with a WR, we might have to poll again after removing
252 err = iwch_poll_cq_one(rhp, chp, wc + npolled);
254 PANIC_IF(++i > 1000);
256 } while (err == -EAGAIN);
260 mtx_unlock(&chp->lock);