1 /**************************************************************************
3 Copyright (c) 2007, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
39 #include <sys/pciio.h>
41 #include <machine/bus.h>
42 #include <machine/resource.h>
43 #include <sys/bus_dma.h>
45 #include <sys/ioccom.h>
47 #include <sys/rwlock.h>
48 #include <sys/linker.h>
49 #include <sys/firmware.h>
50 #include <sys/socket.h>
51 #include <sys/socketvar.h>
52 #include <sys/sockopt.h>
53 #include <sys/sockio.h>
55 #include <sys/sysctl.h>
56 #include <sys/syslog.h>
57 #include <sys/queue.h>
58 #include <sys/taskqueue.h>
60 #include <sys/queue.h>
65 #include <net/route.h>
66 #include <netinet/in_systm.h>
67 #include <netinet/in.h>
68 #include <netinet/in_pcb.h>
69 #include <netinet/ip.h>
70 #include <netinet/ip_var.h>
71 #include <netinet/tcp_var.h>
72 #include <netinet/toecore.h>
73 #include <netinet/tcp.h>
74 #include <netinet/tcpip.h>
76 #include <rdma/ib_verbs.h>
77 #include <linux/idr.h>
78 #include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
80 #include <cxgb_include.h>
81 #include <ulp/tom/cxgb_l2t.h>
82 #include <ulp/tom/cxgb_tom.h>
83 #include <ulp/tom/cxgb_toepcb.h>
84 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
85 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
86 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
87 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
88 #include <ulp/iw_cxgb/iw_cxgb.h>
89 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
91 /* Response queue used for RDMA events. */
92 #define ASYNC_NOTIF_RSPQ 0
94 cxio_rdma_cq_setup(struct cxio_rdev *rdev_p, unsigned id, uint64_t base_addr,
95 unsigned size, unsigned ovfl_mode, unsigned credits, unsigned credit_thres)
97 struct adapter *sc = rdev_p->adap;
100 mtx_lock_spin(&sc->sge.reg_lock);
101 rc = -t3_sge_init_cqcntxt(sc, id, base_addr, size, ASYNC_NOTIF_RSPQ,
102 ovfl_mode, credits, credit_thres);
103 mtx_unlock_spin(&sc->sge.reg_lock);
109 cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,
110 enum t3_cq_opcode op, u32 credit)
115 struct adapter *sc = rdev_p->adap;
117 if (op != CQ_CREDIT_UPDATE)
120 mtx_lock_spin(&sc->sge.reg_lock);
121 ret = t3_sge_cqcntxt_op(sc, cq->cqid, op, credit);
122 mtx_unlock_spin(&sc->sge.reg_lock);
124 if ((ret < 0) || (op == CQ_CREDIT_UPDATE))
128 * If the rearm returned an index other than our current index,
129 * then there might be CQE's in flight (being DMA'd). We must wait
130 * here for them to complete or the consumer can miss a notification.
132 if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {
138 * Keep the generation correct by bumping rptr until it
139 * matches the index returned by the rearm - 1.
141 while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)
145 * Now rptr is the index for the (last) cqe that was
146 * in-flight at the time the HW rearmed the CQ. We
147 * spin until that CQE is valid.
149 cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);
150 while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {
153 struct adapter *sc = rdev_p->adap;
155 log(LOG_ERR, "%s: stalled rnic\n",
156 device_get_nameunit(sc->dev));
169 cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)
172 return (cxio_rdma_cq_setup(rdev_p, cqid, 0, 0, 0, 0, 0));
176 cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
179 struct t3_modify_qp_wr *wqe;
182 m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, wqe);
184 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
187 wqe = mtod(m, struct t3_modify_qp_wr *);
188 memset(wqe, 0, sizeof(*wqe));
189 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);
190 wqe->flags = htobe32(MODQP_WRITE_EC);
191 sge_cmd = qpid << 8 | 3;
192 wqe->sge_cmd = htobe64(sge_cmd);
193 return t3_offload_tx(rdev_p->adap, m);
197 cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel)
199 int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);
201 size += 1; /* one extra page for storing cq-in-err state */
202 cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);
206 cq->sw_queue = malloc(size, M_DEVBUF, M_NOWAIT|M_ZERO);
211 cq->queue = contigmalloc(size,
212 M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
214 cq->dma_addr = vtophys(cq->queue);
216 free(cq->sw_queue, M_DEVBUF);
219 memset(cq->queue, 0, size);
221 return (cxio_rdma_cq_setup(rdev_p, cq->cqid, cq->dma_addr,
222 1UL << cq->size_log2, 0, 65535, 1));
226 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
228 struct cxio_qpid *entry;
232 mtx_lock(&uctx->lock);
233 if (!TAILQ_EMPTY(&uctx->qpids)) {
235 entry = TAILQ_FIRST(&uctx->qpids);
236 TAILQ_REMOVE(&uctx->qpids, entry, entry);
238 free(entry, M_DEVBUF);
240 qpid = cxio_hal_get_qpid(rdev_p->rscp);
243 for (i = qpid+1; i & rdev_p->qpmask; i++) {
244 entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
248 TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
252 mtx_unlock(&uctx->lock);
253 CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
258 put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
259 struct cxio_ucontext *uctx)
261 struct cxio_qpid *entry;
263 entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
264 CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
266 mtx_lock(&uctx->lock);
267 TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
268 mtx_unlock(&uctx->lock);
272 cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
274 struct cxio_qpid *pos, *tmp;
276 mtx_lock(&uctx->lock);
277 TAILQ_FOREACH_SAFE(pos, &uctx->qpids, entry, tmp) {
278 TAILQ_REMOVE(&uctx->qpids, pos, entry);
279 if (!(pos->qpid & rdev_p->qpmask))
280 cxio_hal_put_qpid(rdev_p->rscp, pos->qpid);
283 mtx_unlock(&uctx->lock);
287 cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
289 TAILQ_INIT(&uctx->qpids);
290 mtx_init(&uctx->lock, "cxio uctx", NULL, MTX_DEF|MTX_DUPOK);
294 cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
295 struct t3_wq *wq, struct cxio_ucontext *uctx)
297 int depth = 1UL << wq->size_log2;
298 int rqsize = 1UL << wq->rq_size_log2;
300 wq->qpid = get_qpid(rdev_p, uctx);
304 wq->rq = malloc(depth * sizeof(struct t3_swrq), M_DEVBUF, M_NOWAIT|M_ZERO);
308 wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);
312 wq->sq = malloc(depth * sizeof(struct t3_swsq), M_DEVBUF, M_NOWAIT|M_ZERO);
315 wq->queue = contigmalloc(depth *sizeof(union t3_wr),
316 M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
318 wq->dma_addr = vtophys(wq->queue);
322 memset(wq->queue, 0, depth * sizeof(union t3_wr));
323 wq->doorbell = rdev_p->rnic_info.kdb_addr;
325 wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
326 (wq->qpid << rdev_p->qpshift);
328 CTR4(KTR_IW_CXGB, "%s qpid 0x%x doorbell 0x%p udb 0x%llx", __FUNCTION__,
329 wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
332 free(wq->sq, M_DEVBUF);
334 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);
336 free(wq->rq, M_DEVBUF);
338 put_qpid(rdev_p, wq->qpid, uctx);
343 cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
346 err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
347 free(cq->sw_queue, M_DEVBUF);
349 dma_free_coherent(&(rdev_p->rnic_info.pdev),
350 (1UL << (cq->size_log2))
351 * sizeof(struct t3_cqe), cq->queue,
352 /* pci_unmap_addr(cq, mapping)*/ 0);
354 contigfree(cq->queue,(1UL << (cq->size_log2))
355 * sizeof(struct t3_cqe), M_DEVBUF);
357 cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
362 cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
363 struct cxio_ucontext *uctx)
367 dma_free_coherent(&(rdev_p->rnic_info.pdev),
368 (1UL << (wq->size_log2))
369 * sizeof(union t3_wr), wq->queue,
370 /* pci_unmap_addr(wq, mapping)*/ 0);
372 contigfree(wq->queue, (1UL << (wq->size_log2))
373 * sizeof(union t3_wr), M_DEVBUF);
375 free(wq->sq, M_DEVBUF);
376 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));
377 free(wq->rq, M_DEVBUF);
378 put_qpid(rdev_p, wq->qpid, uctx);
383 insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
387 CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
388 wq, cq, cq->sw_rptr, cq->sw_wptr);
389 memset(&cqe, 0, sizeof(cqe));
390 cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
391 V_CQE_OPCODE(T3_SEND) |
394 V_CQE_QPID(wq->qpid) |
395 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
397 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
402 cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
407 CTR3(KTR_IW_CXGB, "%s wq %p cq %p", __FUNCTION__, wq, cq);
410 CTR4(KTR_IW_CXGB, "%s rq_rptr %u rq_wptr %u skip count %u", __FUNCTION__,
411 wq->rq_rptr, wq->rq_wptr, count);
412 ptr = wq->rq_rptr + count;
413 while (ptr++ != wq->rq_wptr) {
414 insert_recv_cqe(wq, cq);
421 insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
426 CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
427 wq, cq, cq->sw_rptr, cq->sw_wptr);
428 memset(&cqe, 0, sizeof(cqe));
429 cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
430 V_CQE_OPCODE(sqp->opcode) |
433 V_CQE_QPID(wq->qpid) |
434 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
436 cqe.u.scqe.wrid_hi = sqp->sq_wptr;
438 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
443 cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
447 struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
449 ptr = wq->sq_rptr + count;
450 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
451 while (ptr != wq->sq_wptr) {
452 insert_sq_cqe(wq, cq, sqp);
454 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
461 * Move all CQEs from the HWCQ into the SWCQ.
464 cxio_flush_hw_cq(struct t3_cq *cq)
466 struct t3_cqe *cqe, *swcqe;
468 CTR3(KTR_IW_CXGB, "%s cq %p cqid 0x%x", __FUNCTION__, cq, cq->cqid);
469 cqe = cxio_next_hw_cqe(cq);
471 CTR3(KTR_IW_CXGB, "%s flushing hwcq rptr 0x%x to swcq wptr 0x%x",
472 __FUNCTION__, cq->rptr, cq->sw_wptr);
473 swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
475 swcqe->header |= htobe32(V_CQE_SWCQE(1));
478 cqe = cxio_next_hw_cqe(cq);
482 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)
484 if (CQE_OPCODE(*cqe) == T3_TERMINATE)
487 if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe))
490 if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe))
493 if (CQE_OPCODE(*cqe) && RQ_TYPE(*cqe) &&
494 Q_EMPTY(wq->rq_rptr, wq->rq_wptr))
501 cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
508 while (!Q_EMPTY(ptr, cq->sw_wptr)) {
509 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
510 if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) &&
511 (CQE_QPID(*cqe) == wq->qpid))
515 CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
519 cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
525 CTR2(KTR_IW_CXGB, "%s count zero %d", __FUNCTION__, *count);
527 while (!Q_EMPTY(ptr, cq->sw_wptr)) {
528 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
529 if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) &&
530 (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq))
534 CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
538 cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)
541 return (cxio_rdma_cq_setup(rdev_p, 0, 0, 1, 1, 0, 0));
545 cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
548 u64 sge_cmd, ctx0, ctx1;
550 struct t3_modify_qp_wr *wqe;
553 m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, wqe);
555 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
558 err = cxio_hal_init_ctrl_cq(rdev_p);
560 CTR2(KTR_IW_CXGB, "%s err %d initializing ctrl_cq", __FUNCTION__, err);
564 rdev_p->ctrl_qp.workq = contigmalloc((1 << T3_CTRL_QP_SIZE_LOG2)
565 *sizeof(union t3_wr), M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
566 if (rdev_p->ctrl_qp.workq)
567 rdev_p->ctrl_qp.dma_addr = vtophys(rdev_p->ctrl_qp.workq);
569 CTR1(KTR_IW_CXGB, "%s dma_alloc_coherent failed", __FUNCTION__);
574 rdev_p->ctrl_qp.doorbell = rdev_p->rnic_info.kdb_addr;
575 memset(rdev_p->ctrl_qp.workq, 0,
576 (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));
578 mtx_init(&rdev_p->ctrl_qp.lock, "ctl-qp lock", NULL, MTX_DEF|MTX_DUPOK);
580 /* update HW Ctrl QP context */
581 base_addr = rdev_p->ctrl_qp.dma_addr;
583 ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) |
584 V_EC_BASE_LO((u32) base_addr & 0xffff));
586 ctx0 |= V_EC_CREDITS(FW_WR_NUM);
588 ctx1 = (u32) base_addr;
590 ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) |
591 V_EC_TYPE(0) | V_EC_GEN(1) |
592 V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) << 32;
593 memset(wqe, 0, sizeof(*wqe));
594 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0,
596 wqe->flags = htobe32(MODQP_WRITE_EC);
597 sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
598 wqe->sge_cmd = htobe64(sge_cmd);
599 wqe->ctx1 = htobe64(ctx1);
600 wqe->ctx0 = htobe64(ctx0);
601 CTR3(KTR_IW_CXGB, "CtrlQP dma_addr 0x%llx workq %p size %d",
602 (unsigned long long) rdev_p->ctrl_qp.dma_addr,
603 rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
604 return t3_offload_tx(rdev_p->adap, m);
611 cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
615 dma_free_coherent(&(rdev_p->rnic_info.pdev),
616 (1UL << T3_CTRL_QP_SIZE_LOG2)
617 * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,
618 /* pci_unmap_addr(&rdev_p->ctrl_qp, mapping)*/ 0);
620 contigfree(rdev_p->ctrl_qp.workq,(1UL << T3_CTRL_QP_SIZE_LOG2)
621 * sizeof(union t3_wr), M_DEVBUF);
623 return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);
626 /* write len bytes of data into addr (32B aligned address)
627 * If data is NULL, clear len byte of memory to zero.
628 * caller aquires the ctrl_qp lock before the call
631 cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
634 u32 i, nr_wqe, copy_len;
636 u8 wr_len, utx_len; /* lenght in 8 byte flit */
637 enum t3_wr_flags flag;
641 nr_wqe = len % 96 ? len / 96 + 1 : len / 96; /* 96B max per WQE */
642 CTR6(KTR_IW_CXGB, "cxio_hal_ctrl_qp_write_mem wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x",
643 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,
645 utx_len = 3; /* in 32B unit */
646 for (i = 0; i < nr_wqe; i++) {
647 if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,
648 T3_CTRL_QP_SIZE_LOG2)) {
649 CTR4(KTR_IW_CXGB, "%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, "
650 "wait for more space i %d", __FUNCTION__,
651 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);
652 if (cxio_wait(&rdev_p->ctrl_qp,
653 &rdev_p->ctrl_qp.lock,
654 !Q_FULL(rdev_p->ctrl_qp.rptr,
655 rdev_p->ctrl_qp.wptr,
656 T3_CTRL_QP_SIZE_LOG2))) {
657 CTR1(KTR_IW_CXGB, "%s ctrl_qp workq interrupted",
661 CTR2(KTR_IW_CXGB, "%s ctrl_qp wakeup, continue posting work request "
662 "i %d", __FUNCTION__, i);
664 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
665 (1 << T3_CTRL_QP_SIZE_LOG2)));
667 if (i == (nr_wqe - 1)) {
669 flag = T3_COMPLETION_FLAG;
671 utx_len = len / 32 + 1;
677 * Force a CQE to return the credit to the workq in case
678 * we posted more than half the max QP size of WRs
681 (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) {
682 flag = T3_COMPLETION_FLAG;
683 CTR2(KTR_IW_CXGB, "%s force completion at i %d", __FUNCTION__, i);
686 /* build the utx mem command */
687 wqe += (sizeof(struct t3_bypass_wr) >> 3);
688 utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3);
690 utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1);
691 *wqe = htobe64(utx_cmd);
693 copy_data = (u8 *) data + i * 96;
694 copy_len = len > 96 ? 96 : len;
696 /* clear memory content if data is NULL */
698 memcpy(wqe, copy_data, copy_len);
700 memset(wqe, 0, copy_len);
702 memset(((u8 *) wqe) + copy_len, 0,
703 32 - (copy_len % 32));
704 wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 +
706 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
707 (1 << T3_CTRL_QP_SIZE_LOG2)));
709 /* wptr in the WRID[31:0] */
710 ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;
713 * This must be the last write with a memory barrier
716 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
717 Q_GENBIT(rdev_p->ctrl_qp.wptr,
718 T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,
720 if (flag == T3_COMPLETION_FLAG)
721 ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
724 rdev_p->ctrl_qp.wptr++;
729 /* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size
730 * OUT: stag index, actual pbl_size, pbl_addr allocated.
731 * TBD: shared memory region support
734 __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
735 u32 *stag, u8 stag_state, u32 pdid,
736 enum tpt_mem_type type, enum tpt_mem_perm perm,
737 u32 zbva, u64 to, u32 len, u8 page_size,
738 u32 pbl_size, u32 pbl_addr)
741 struct tpt_entry tpt;
745 stag_state = stag_state > 0;
746 stag_idx = (*stag) >> 8;
748 if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) {
749 stag_idx = cxio_hal_get_stag(rdev_p->rscp);
752 *stag = (stag_idx << 8) | ((*stag) & 0xFF);
754 CTR5(KTR_IW_CXGB, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x",
755 __FUNCTION__, stag_state, type, pdid, stag_idx);
757 mtx_lock(&rdev_p->ctrl_qp.lock);
759 /* write TPT entry */
761 memset(&tpt, 0, sizeof(tpt));
763 tpt.valid_stag_pdid = htobe32(F_TPT_VALID |
764 V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) |
765 V_TPT_STAG_STATE(stag_state) |
766 V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));
767 PANIC_IF(page_size >= 28);
768 tpt.flags_pagesize_qpid = htobe32(V_TPT_PERM(perm) |
769 F_TPT_MW_BIND_ENABLE |
770 V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
771 V_TPT_PAGE_SIZE(page_size));
772 tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
773 htobe32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)>>3));
774 tpt.len = htobe32(len);
775 tpt.va_hi = htobe32((u32) (to >> 32));
776 tpt.va_low_or_fbo = htobe32((u32) (to & 0xFFFFFFFFULL));
777 tpt.rsvd_bind_cnt_or_pstag = 0;
778 tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
779 htobe32(V_TPT_PBL_SIZE((pbl_size) >> 2));
781 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
783 (rdev_p->rnic_info.tpt_base >> 5),
786 /* release the stag index to free pool */
788 cxio_hal_put_stag(rdev_p->rscp, stag_idx);
790 wptr = rdev_p->ctrl_qp.wptr;
791 mtx_unlock(&rdev_p->ctrl_qp.lock);
793 if (cxio_wait(&rdev_p->ctrl_qp,
794 &rdev_p->ctrl_qp.lock,
795 SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr)))
800 int cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl,
801 u32 pbl_addr, u32 pbl_size)
806 CTR4(KTR_IW_CXGB, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d",
807 __func__, pbl_addr, rdev_p->rnic_info.pbl_base,
810 mtx_lock(&rdev_p->ctrl_qp.lock);
811 err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr >> 5, pbl_size << 3,
813 wptr = rdev_p->ctrl_qp.wptr;
814 mtx_unlock(&rdev_p->ctrl_qp.lock);
818 if (cxio_wait(&rdev_p->ctrl_qp,
819 &rdev_p->ctrl_qp.lock,
820 SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr)))
827 cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
828 enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
829 u8 page_size, u32 pbl_size, u32 pbl_addr)
831 *stag = T3_STAG_UNSET;
832 return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
833 zbva, to, len, page_size, pbl_size, pbl_addr);
837 cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
838 enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
839 u8 page_size, u32 pbl_size, u32 pbl_addr)
841 return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
842 zbva, to, len, page_size, pbl_size, pbl_addr);
846 cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
849 return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,
854 cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
856 *stag = T3_STAG_UNSET;
857 return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
862 cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
864 return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0,
869 cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr,
872 struct t3_rdma_init_wr *wqe;
880 m = M_GETHDR_OFLD(0, CPL_PRIORITY_DATA, wqe);
883 CTR2(KTR_IW_CXGB, "%s rdev_p %p", __FUNCTION__, rdev_p);
884 wqe->wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_INIT));
885 wqe->wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(attr->tid) |
886 V_FW_RIWR_LEN(sizeof(*wqe) >> 3));
888 wqe->qpid = htobe32(attr->qpid);
889 wqe->pdid = htobe32(attr->pdid);
890 wqe->scqid = htobe32(attr->scqid);
891 wqe->rcqid = htobe32(attr->rcqid);
892 wqe->rq_addr = htobe32(attr->rq_addr - rdev_p->rnic_info.rqt_base);
893 wqe->rq_size = htobe32(attr->rq_size);
894 wqe->mpaattrs = attr->mpaattrs;
895 wqe->qpcaps = attr->qpcaps;
896 wqe->ulpdu_size = htobe16(attr->tcp_emss);
897 wqe->rqe_count = htobe16(attr->rqe_count);
898 wqe->flags_rtr_type = htobe16(attr->flags |
899 V_RTR_TYPE(attr->rtr_type) |
901 wqe->ord = htobe32(attr->ord);
902 wqe->ird = htobe32(attr->ird);
903 wqe->qp_dma_addr = htobe64(attr->qp_dma_addr);
904 wqe->qp_dma_size = htobe32(attr->qp_dma_size);
905 wqe->irs = htobe32(attr->irs);
907 /* XXX: bad form, fix later */
912 oh = mtod(m, struct ofld_hdr *);
914 oh->flags |= F_HDR_DF;
917 toep->tp_wr_unacked++;
918 rc = t3_offload_tx(rdev_p->adap, m);
925 cxio_hal_ev_handler(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m)
927 struct adapter *sc = qs->adap;
928 struct iwch_dev *rnicp = sc->iwarp_softc;
929 struct cxio_rdev *rdev_p = &rnicp->rdev;
930 struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data;
931 int qpid = CQE_QPID(rsp_msg->cqe);
933 CTR6(KTR_IW_CXGB, "%s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x",
934 __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
935 RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg));
936 CTR4(KTR_IW_CXGB, "se %0x notify %0x cqbranch %0x creditth %0x",
937 RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
938 RSPQ_CREDIT_THRESH(rsp_msg));
939 CTR4(KTR_IW_CXGB, "CQE: QPID 0x%0x type 0x%0x status 0x%0x opcode %d",
940 qpid, CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
941 CQE_OPCODE(rsp_msg->cqe));
942 CTR3(KTR_IW_CXGB, "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
943 CQE_LEN(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
947 mtx_lock(&rdev_p->ctrl_qp.lock);
948 rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
949 wakeup(&rdev_p->ctrl_qp);
950 mtx_unlock(&rdev_p->ctrl_qp.lock);
955 iwch_ev_dispatch(rnicp, m);
962 /* Caller takes care of locking if needed */
964 cxio_rdev_open(struct cxio_rdev *rdev_p)
967 struct rdma_info *ri = &rdev_p->rnic_info;
968 struct adapter *sc = rdev_p->adap;
970 KASSERT(rdev_p->adap, ("%s: adap is NULL", __func__));
972 memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
974 ri->udbell_physbase = rman_get_start(sc->udbs_res);
975 ri->udbell_len = rman_get_size(sc->udbs_res);
976 ri->tpt_base = t3_read_reg(sc, A_ULPTX_TPT_LLIMIT);
977 ri->tpt_top = t3_read_reg(sc, A_ULPTX_TPT_ULIMIT);
978 ri->pbl_base = t3_read_reg(sc, A_ULPTX_PBL_LLIMIT);
979 ri->pbl_top = t3_read_reg(sc, A_ULPTX_PBL_ULIMIT);
980 ri->rqt_base = t3_read_reg(sc, A_ULPRX_RQ_LLIMIT);
981 ri->rqt_top = t3_read_reg(sc, A_ULPRX_RQ_ULIMIT);
982 ri->kdb_addr = (void *)((unsigned long)
983 rman_get_virtual(sc->regs_res) + A_SG_KDOORBELL);
986 * qpshift is the number of bits to shift the qpid left in order
987 * to get the correct address of the doorbell for that qp.
989 cxio_init_ucontext(rdev_p, &rdev_p->uctx);
990 rdev_p->qpshift = PAGE_SHIFT -
992 ilog2(rdev_p->rnic_info.udbell_len >>
994 rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
995 rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
996 CTR4(KTR_IW_CXGB, "cxio_rdev_open rnic %p info: tpt_base 0x%0x tpt_top 0x%0x num stags %d",
997 rdev_p->adap, rdev_p->rnic_info.tpt_base,
998 rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p));
999 CTR4(KTR_IW_CXGB, "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x",
1000 rdev_p->rnic_info.pbl_base,
1001 rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
1002 rdev_p->rnic_info.rqt_top);
1003 CTR6(KTR_IW_CXGB, "udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
1004 "qpnr %d qpmask 0x%x",
1005 rdev_p->rnic_info.udbell_len,
1006 rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
1007 rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);
1009 err = cxio_hal_init_ctrl_qp(rdev_p);
1011 log(LOG_ERR, "%s error %d initializing ctrl_qp.\n",
1015 err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
1016 0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,
1019 log(LOG_ERR, "%s error %d initializing hal resources.\n",
1023 err = cxio_hal_pblpool_create(rdev_p);
1025 log(LOG_ERR, "%s error %d initializing pbl mem pool.\n",
1029 err = cxio_hal_rqtpool_create(rdev_p);
1031 log(LOG_ERR, "%s error %d initializing rqt mem pool.\n",
1037 cxio_hal_pblpool_destroy(rdev_p);
1039 cxio_hal_destroy_resource(rdev_p->rscp);
1041 cxio_hal_destroy_ctrl_qp(rdev_p);
1047 cxio_rdev_close(struct cxio_rdev *rdev_p)
1049 cxio_hal_pblpool_destroy(rdev_p);
1050 cxio_hal_rqtpool_destroy(rdev_p);
1051 cxio_hal_destroy_ctrl_qp(rdev_p);
1052 cxio_hal_destroy_resource(rdev_p->rscp);
1056 cxio_hal_init(struct adapter *sc)
1059 if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))
1062 t3_register_cpl_handler(sc, CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
1068 cxio_hal_uninit(struct adapter *sc)
1070 t3_register_cpl_handler(sc, CPL_ASYNC_NOTIF, NULL);
1072 cxio_hal_destroy_rhdl_resource();
1077 flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
1079 struct t3_swsq *sqp;
1080 __u32 ptr = wq->sq_rptr;
1081 int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
1083 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1085 if (!sqp->signaled) {
1087 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1088 } else if (sqp->complete) {
1091 * Insert this completed cqe into the swcq.
1093 CTR3(KTR_IW_CXGB, "%s moving cqe into swcq sq idx %ld cq idx %ld",
1094 __FUNCTION__, Q_PTR2IDX(ptr, wq->sq_size_log2),
1095 Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
1096 sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
1097 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
1107 create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
1108 struct t3_cqe *read_cqe)
1110 read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
1111 read_cqe->len = wq->oldest_read->read_len;
1112 read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
1113 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
1114 V_CQE_OPCODE(T3_READ_REQ) |
1119 * Return a ptr to the next read wr in the SWSQ or NULL.
1122 advance_oldest_read(struct t3_wq *wq)
1125 u32 rptr = wq->oldest_read - wq->sq + 1;
1126 u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
1128 while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
1129 wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
1131 if (wq->oldest_read->opcode == T3_READ_REQ)
1135 wq->oldest_read = NULL;
1142 * check the validity of the first CQE,
1143 * supply the wq assicated with the qpid.
1145 * credit: cq credit to return to sge.
1146 * cqe_flushed: 1 iff the CQE is flushed.
1147 * cqe: copy of the polled CQE.
1151 * -1 CQE skipped, try again.
1154 cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
1155 u8 *cqe_flushed, u64 *cookie, u32 *credit)
1158 struct t3_cqe *hw_cqe, read_cqe;
1162 hw_cqe = cxio_next_cqe(cq);
1164 CTR5(KTR_IW_CXGB, "cxio_poll_cq CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x",
1165 CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
1166 CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe));
1167 CTR4(KTR_IW_CXGB, "opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
1168 CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
1169 CQE_WRID_LOW(*hw_cqe));
1172 * skip cqe's not affiliated with a QP.
1180 * Gotta tweak READ completions:
1181 * 1) the cqe doesn't contain the sq_wptr from the wr.
1182 * 2) opcode not reflected from the wr.
1183 * 3) read_len not reflected from the wr.
1184 * 4) cq_type is RQ_TYPE not SQ_TYPE.
1186 if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
1189 * Don't write to the HWCQ, so create a new read req CQE
1192 create_read_req_cqe(wq, hw_cqe, &read_cqe);
1194 advance_oldest_read(wq);
1198 * T3A: Discard TERMINATE CQEs.
1200 if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
1206 if (CQE_STATUS(*hw_cqe) || wq->error) {
1207 *cqe_flushed = wq->error;
1211 * T3A inserts errors into the CQE. We cannot return
1212 * these as work completions.
1214 /* incoming write failures */
1215 if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
1216 && RQ_TYPE(*hw_cqe)) {
1220 /* incoming read request failures */
1221 if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
1226 /* incoming SEND with no receive posted failures */
1227 if (CQE_OPCODE(*hw_cqe) && RQ_TYPE(*hw_cqe) &&
1228 Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1232 PANIC_IF((*cqe_flushed == 0) && !SW_CQE(*hw_cqe));
1239 if (RQ_TYPE(*hw_cqe)) {
1242 * HW only validates 4 bits of MSN. So we must validate that
1243 * the MSN in the SEND is the next expected MSN. If its not,
1244 * then we complete this with TPT_ERR_MSN and mark the wq in
1248 if (Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1254 if (__predict_false((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
1256 hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
1263 * If we get here its a send completion.
1265 * Handle out of order completion. These get stuffed
1266 * in the SW SQ. Then the SW SQ is walked to move any
1267 * now in-order completions into the SW CQ. This handles
1269 * 1) reaping unsignaled WRs when the first subsequent
1270 * signaled WR is completed.
1271 * 2) out of order read completions.
1273 if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
1274 struct t3_swsq *sqp;
1276 CTR2(KTR_IW_CXGB, "%s out of order completion going in swsq at idx %ld",
1278 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
1280 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
1291 * Reap the associated WR(s) that are freed up with this
1294 if (SQ_TYPE(*hw_cqe)) {
1295 wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
1296 CTR2(KTR_IW_CXGB, "%s completing sq idx %ld", __FUNCTION__,
1297 Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
1298 *cookie = wq->sq[Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)].wr_id;
1301 CTR2(KTR_IW_CXGB, "%s completing rq idx %ld", __FUNCTION__,
1302 Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1303 *cookie = wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].wr_id;
1304 if (wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].pbl_addr)
1305 cxio_hal_pblpool_free(wq->rdev,
1306 wq->rq[Q_PTR2IDX(wq->rq_rptr,
1307 wq->rq_size_log2)].pbl_addr, T3_STAG0_PBL_SIZE);
1308 PANIC_IF(Q_EMPTY(wq->rq_rptr, wq->rq_wptr));
1314 * Flush any completed cqes that are now in-order.
1316 flush_completed_wrs(wq, cq);
1319 if (SW_CQE(*hw_cqe)) {
1320 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x",
1321 __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
1324 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip hw cqe rptr 0x%x",
1325 __FUNCTION__, cq, cq->cqid, cq->rptr);
1329 * T3A: compute credits.
1331 if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
1332 || ((cq->rptr - cq->wptr) >= 128)) {
1333 *credit = cq->rptr - cq->wptr;
1334 cq->wptr = cq->rptr;