2 /**************************************************************************
4 Copyright (c) 2007, Chelsio Inc.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
29 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/pciio.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/bus_dma.h>
44 #include <sys/ioccom.h>
46 #include <sys/rwlock.h>
47 #include <sys/linker.h>
48 #include <sys/firmware.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/queue.h>
55 #include <sys/taskqueue.h>
57 #include <sys/queue.h>
62 #include <netinet/in.h>
64 #include <contrib/rdma/ib_verbs.h>
66 #include <cxgb_include.h>
67 #include <ulp/tom/cxgb_l2t.h>
68 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
69 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
70 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
71 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
72 #include <ulp/iw_cxgb/iw_cxgb.h>
73 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
75 static TAILQ_HEAD( ,cxio_rdev) rdev_list;
76 static cxio_hal_ev_callback_func_t cxio_ev_cb = NULL;
78 static struct cxio_rdev *
79 cxio_hal_find_rdev_by_name(char *dev_name)
81 struct cxio_rdev *rdev;
83 TAILQ_FOREACH(rdev, &rdev_list, entry)
84 if (!strcmp(rdev->dev_name, dev_name))
90 cxio_hal_find_rdev_by_t3cdev(struct t3cdev *tdev)
92 struct cxio_rdev *rdev;
94 TAILQ_FOREACH(rdev, &rdev_list, entry)
95 if (rdev->t3cdev_p == tdev)
101 cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,
102 enum t3_cq_opcode op, u32 credit)
108 struct rdma_cq_op setup;
110 setup.credits = (op == CQ_CREDIT_UPDATE) ? credit : 0;
112 ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);
114 if ((ret < 0) || (op == CQ_CREDIT_UPDATE))
118 * If the rearm returned an index other than our current index,
119 * then there might be CQE's in flight (being DMA'd). We must wait
120 * here for them to complete or the consumer can miss a notification.
122 if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {
128 * Keep the generation correct by bumping rptr until it
129 * matches the index returned by the rearm - 1.
131 while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)
135 * Now rptr is the index for the (last) cqe that was
136 * in-flight at the time the HW rearmed the CQ. We
137 * spin until that CQE is valid.
139 cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);
140 while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {
144 log(LOG_ERR, "%s: stalled rnic\n",
157 cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)
159 struct rdma_cq_setup setup;
161 setup.base_addr = 0; /* NULL address */
162 setup.size = 0; /* disaable the CQ */
164 setup.credit_thres = 0;
166 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
170 cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
173 struct t3_modify_qp_wr *wqe;
174 struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
176 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
179 wqe = mtod(m, struct t3_modify_qp_wr *);
180 m->m_len = m->m_pkthdr.len = sizeof(*wqe);
181 memset(wqe, 0, sizeof(*wqe));
182 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);
183 wqe->flags = htobe32(MODQP_WRITE_EC);
184 sge_cmd = qpid << 8 | 3;
185 wqe->sge_cmd = htobe64(sge_cmd);
186 m_set_priority(m, CPL_PRIORITY_CONTROL);
189 return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
193 cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
195 struct rdma_cq_setup setup;
196 int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);
198 cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);
201 cq->sw_queue = malloc(size, M_DEVBUF, M_NOWAIT|M_ZERO);
205 cq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
206 (1UL << (cq->size_log2)) *
207 sizeof(struct t3_cqe),
208 &(cq->dma_addr), M_NOWAIT);
210 cq->queue = contigmalloc((1UL << (cq->size_log2))*sizeof(struct t3_cqe),
211 M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
213 cq->dma_addr = vtophys(cq->queue);
215 free(cq->sw_queue, M_DEVBUF);
221 pci_unmap_addr_set(cq, mapping, cq->dma_addr);
223 memset(cq->queue, 0, size);
225 setup.base_addr = (u64) (cq->dma_addr);
226 setup.size = 1UL << cq->size_log2;
227 setup.credits = 65535;
228 setup.credit_thres = 1;
229 if (rdev_p->t3cdev_p->type != T3A)
233 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
237 cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
239 struct rdma_cq_setup setup;
241 setup.base_addr = (u64) (cq->dma_addr);
242 setup.size = 1UL << cq->size_log2;
243 setup.credits = setup.size;
244 setup.credit_thres = setup.size; /* TBD: overflow recovery */
246 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
250 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
252 struct cxio_qpid *entry;
256 mtx_lock(&uctx->lock);
257 if (!TAILQ_EMPTY(&uctx->qpids)) {
259 entry = TAILQ_FIRST(&uctx->qpids);
260 TAILQ_REMOVE(&uctx->qpids, entry, entry);
262 free(entry, M_DEVBUF);
264 qpid = cxio_hal_get_qpid(rdev_p->rscp);
267 for (i = qpid+1; i & rdev_p->qpmask; i++) {
268 entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
272 TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
276 mtx_unlock(&uctx->lock);
277 CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
282 put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
283 struct cxio_ucontext *uctx)
285 struct cxio_qpid *entry;
287 entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
288 CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
290 mtx_lock(&uctx->lock);
291 TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
292 mtx_unlock(&uctx->lock);
296 cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
298 struct cxio_qpid *pos, *tmp;
300 mtx_lock(&uctx->lock);
301 TAILQ_FOREACH_SAFE(pos, &uctx->qpids, entry, tmp) {
302 TAILQ_REMOVE(&uctx->qpids, pos, entry);
303 if (!(pos->qpid & rdev_p->qpmask))
304 cxio_hal_put_qpid(rdev_p->rscp, pos->qpid);
307 mtx_unlock(&uctx->lock);
311 cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
313 TAILQ_INIT(&uctx->qpids);
314 mtx_init(&uctx->lock, "cxio uctx", NULL, MTX_DEF|MTX_DUPOK);
318 cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
319 struct t3_wq *wq, struct cxio_ucontext *uctx)
321 int depth = 1UL << wq->size_log2;
322 int rqsize = 1UL << wq->rq_size_log2;
324 wq->qpid = get_qpid(rdev_p, uctx);
328 wq->rq = malloc(depth * sizeof(u64), M_DEVBUF, M_NOWAIT|M_ZERO);
332 wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);
336 wq->sq = malloc(depth * sizeof(struct t3_swsq), M_DEVBUF, M_NOWAIT|M_ZERO);
340 wq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
341 depth * sizeof(union t3_wr),
342 &(wq->dma_addr), M_NOWAIT);
344 wq->queue = contigmalloc(depth *sizeof(union t3_wr),
345 M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
347 wq->dma_addr = vtophys(wq->queue);
353 memset(wq->queue, 0, depth * sizeof(union t3_wr));
355 pci_unmap_addr_set(wq, mapping, wq->dma_addr);
357 wq->doorbell = rdev_p->rnic_info.kdb_addr;
359 wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
360 (wq->qpid << rdev_p->qpshift);
361 CTR4(KTR_IW_CXGB, "%s qpid 0x%x doorbell 0x%p udb 0x%llx", __FUNCTION__,
362 wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
365 free(wq->sq, M_DEVBUF);
367 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);
369 free(wq->rq, M_DEVBUF);
371 put_qpid(rdev_p, wq->qpid, uctx);
376 cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
379 err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
380 free(cq->sw_queue, M_DEVBUF);
382 dma_free_coherent(&(rdev_p->rnic_info.pdev),
383 (1UL << (cq->size_log2))
384 * sizeof(struct t3_cqe), cq->queue,
385 /* pci_unmap_addr(cq, mapping)*/ 0);
387 contigfree(cq->queue,(1UL << (cq->size_log2))
388 * sizeof(struct t3_cqe), M_DEVBUF);
390 cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
395 cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
396 struct cxio_ucontext *uctx)
400 dma_free_coherent(&(rdev_p->rnic_info.pdev),
401 (1UL << (wq->size_log2))
402 * sizeof(union t3_wr), wq->queue,
403 /* pci_unmap_addr(wq, mapping)*/ 0);
405 contigfree(wq->queue, (1UL << (wq->size_log2))
406 * sizeof(union t3_wr), M_DEVBUF);
408 free(wq->sq, M_DEVBUF);
409 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));
410 free(wq->rq, M_DEVBUF);
411 put_qpid(rdev_p, wq->qpid, uctx);
416 insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
420 CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
421 wq, cq, cq->sw_rptr, cq->sw_wptr);
422 memset(&cqe, 0, sizeof(cqe));
423 cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
424 V_CQE_OPCODE(T3_SEND) |
427 V_CQE_QPID(wq->qpid) |
428 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
430 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
435 cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
439 CTR3(KTR_IW_CXGB, "%s wq %p cq %p", __FUNCTION__, wq, cq);
442 CTR4(KTR_IW_CXGB, "%s rq_rptr %u rq_wptr %u skip count %u", __FUNCTION__,
443 wq->rq_rptr, wq->rq_wptr, count);
444 ptr = wq->rq_rptr + count;
445 while (ptr++ != wq->rq_wptr)
446 insert_recv_cqe(wq, cq);
450 insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
455 CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
456 wq, cq, cq->sw_rptr, cq->sw_wptr);
457 memset(&cqe, 0, sizeof(cqe));
458 cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
459 V_CQE_OPCODE(sqp->opcode) |
462 V_CQE_QPID(wq->qpid) |
463 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
465 cqe.u.scqe.wrid_hi = sqp->sq_wptr;
467 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
472 cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
475 struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
477 ptr = wq->sq_rptr + count;
479 while (ptr != wq->sq_wptr) {
480 insert_sq_cqe(wq, cq, sqp);
487 * Move all CQEs from the HWCQ into the SWCQ.
490 cxio_flush_hw_cq(struct t3_cq *cq)
492 struct t3_cqe *cqe, *swcqe;
494 CTR3(KTR_IW_CXGB, "%s cq %p cqid 0x%x", __FUNCTION__, cq, cq->cqid);
495 cqe = cxio_next_hw_cqe(cq);
497 CTR3(KTR_IW_CXGB, "%s flushing hwcq rptr 0x%x to swcq wptr 0x%x",
498 __FUNCTION__, cq->rptr, cq->sw_wptr);
499 swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
501 swcqe->header |= htobe32(V_CQE_SWCQE(1));
504 cqe = cxio_next_hw_cqe(cq);
508 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)
510 if (CQE_OPCODE(*cqe) == T3_TERMINATE)
513 if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe))
516 if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe))
519 if ((CQE_OPCODE(*cqe) == T3_SEND) && RQ_TYPE(*cqe) &&
520 Q_EMPTY(wq->rq_rptr, wq->rq_wptr))
527 cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
534 while (!Q_EMPTY(ptr, cq->sw_wptr)) {
535 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
536 if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) &&
537 (CQE_QPID(*cqe) == wq->qpid))
541 CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
545 cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
551 CTR2(KTR_IW_CXGB, "%s count zero %d", __FUNCTION__, *count);
553 while (!Q_EMPTY(ptr, cq->sw_wptr)) {
554 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
555 if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) &&
556 (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq))
560 CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
564 cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)
566 struct rdma_cq_setup setup;
568 setup.base_addr = 0; /* NULL address */
569 setup.size = 1; /* enable the CQ */
572 /* force SGE to redirect to RspQ and interrupt */
573 setup.credit_thres = 0;
575 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
579 cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
582 u64 sge_cmd, ctx0, ctx1;
584 struct t3_modify_qp_wr *wqe;
587 m = m_gethdr(MT_DATA, M_NOWAIT);
589 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
592 err = cxio_hal_init_ctrl_cq(rdev_p);
594 CTR2(KTR_IW_CXGB, "%s err %d initializing ctrl_cq", __FUNCTION__, err);
598 rdev_p->ctrl_qp.workq = dma_alloc_coherent(
599 rdev_p->rnic_info.pdev,
600 (1 << T3_CTRL_QP_SIZE_LOG2) *
602 &(rdev_p->ctrl_qp.dma_addr),
605 rdev_p->ctrl_qp.workq = contigmalloc((1 << T3_CTRL_QP_SIZE_LOG2)
606 *sizeof(union t3_wr), M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
607 if (rdev_p->ctrl_qp.workq)
608 rdev_p->ctrl_qp.dma_addr = vtophys(rdev_p->ctrl_qp.workq);
612 if (!rdev_p->ctrl_qp.workq) {
613 CTR1(KTR_IW_CXGB, "%s dma_alloc_coherent failed", __FUNCTION__);
618 pci_unmap_addr_set(&rdev_p->ctrl_qp, mapping,
619 rdev_p->ctrl_qp.dma_addr);
621 rdev_p->ctrl_qp.doorbell = (void /*__iomem */ *)rdev_p->rnic_info.kdb_addr;
622 memset(rdev_p->ctrl_qp.workq, 0,
623 (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));
625 mtx_init(&rdev_p->ctrl_qp.lock, "ctl-qp lock", NULL, MTX_DEF|MTX_DUPOK);
627 /* update HW Ctrl QP context */
628 base_addr = rdev_p->ctrl_qp.dma_addr;
630 ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) |
631 V_EC_BASE_LO((u32) base_addr & 0xffff));
633 ctx0 |= V_EC_CREDITS(FW_WR_NUM);
635 ctx1 = (u32) base_addr;
637 ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) |
638 V_EC_TYPE(0) | V_EC_GEN(1) |
639 V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) << 32;
640 wqe = mtod(m, struct t3_modify_qp_wr *);
641 m->m_len = m->m_pkthdr.len = sizeof(*wqe);
642 memset(wqe, 0, sizeof(*wqe));
643 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0,
645 wqe->flags = htobe32(MODQP_WRITE_EC);
646 sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
647 wqe->sge_cmd = htobe64(sge_cmd);
648 wqe->ctx1 = htobe64(ctx1);
649 wqe->ctx0 = htobe64(ctx0);
650 CTR3(KTR_IW_CXGB, "CtrlQP dma_addr 0x%llx workq %p size %d",
651 (unsigned long long) rdev_p->ctrl_qp.dma_addr,
652 rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
653 m_set_priority(m, CPL_PRIORITY_CONTROL);
656 return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
663 cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
667 dma_free_coherent(&(rdev_p->rnic_info.pdev),
668 (1UL << T3_CTRL_QP_SIZE_LOG2)
669 * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,
670 /* pci_unmap_addr(&rdev_p->ctrl_qp, mapping)*/ 0);
672 contigfree(rdev_p->ctrl_qp.workq,(1UL << T3_CTRL_QP_SIZE_LOG2)
673 * sizeof(union t3_wr), M_DEVBUF);
675 return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);
678 /* write len bytes of data into addr (32B aligned address)
679 * If data is NULL, clear len byte of memory to zero.
680 * caller aquires the ctrl_qp lock before the call
683 cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
684 u32 len, void *data, int completion)
686 u32 i, nr_wqe, copy_len;
688 u8 wr_len, utx_len; /* lenght in 8 byte flit */
689 enum t3_wr_flags flag;
693 nr_wqe = len % 96 ? len / 96 + 1 : len / 96; /* 96B max per WQE */
694 CTR6(KTR_IW_CXGB, "cxio_hal_ctrl_qp_write_mem wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x",
695 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,
697 utx_len = 3; /* in 32B unit */
698 for (i = 0; i < nr_wqe; i++) {
699 if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,
700 T3_CTRL_QP_SIZE_LOG2)) {
701 CTR4(KTR_IW_CXGB, "%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, "
702 "wait for more space i %d", __FUNCTION__,
703 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);
704 if (cxio_wait(&rdev_p->ctrl_qp,
705 &rdev_p->ctrl_qp.lock,
706 !Q_FULL(rdev_p->ctrl_qp.rptr,
707 rdev_p->ctrl_qp.wptr,
708 T3_CTRL_QP_SIZE_LOG2))) {
709 CTR1(KTR_IW_CXGB, "%s ctrl_qp workq interrupted",
713 CTR2(KTR_IW_CXGB, "%s ctrl_qp wakeup, continue posting work request "
714 "i %d", __FUNCTION__, i);
716 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
717 (1 << T3_CTRL_QP_SIZE_LOG2)));
719 if (i == (nr_wqe - 1)) {
721 flag = completion ? T3_COMPLETION_FLAG : 0;
723 utx_len = len / 32 + 1;
729 * Force a CQE to return the credit to the workq in case
730 * we posted more than half the max QP size of WRs
733 (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) {
734 flag = T3_COMPLETION_FLAG;
735 CTR2(KTR_IW_CXGB, "%s force completion at i %d", __FUNCTION__, i);
738 /* build the utx mem command */
739 wqe += (sizeof(struct t3_bypass_wr) >> 3);
740 utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3);
742 utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1);
743 *wqe = htobe64(utx_cmd);
745 copy_data = (u8 *) data + i * 96;
746 copy_len = len > 96 ? 96 : len;
748 /* clear memory content if data is NULL */
750 memcpy(wqe, copy_data, copy_len);
752 memset(wqe, 0, copy_len);
754 memset(((u8 *) wqe) + copy_len, 0,
755 32 - (copy_len % 32));
756 wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 +
758 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
759 (1 << T3_CTRL_QP_SIZE_LOG2)));
761 /* wptr in the WRID[31:0] */
762 ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;
765 * This must be the last write with a memory barrier
768 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
769 Q_GENBIT(rdev_p->ctrl_qp.wptr,
770 T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,
772 if (flag == T3_COMPLETION_FLAG)
773 ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
776 rdev_p->ctrl_qp.wptr++;
781 /* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size
782 * OUT: stag index, actual pbl_size, pbl_addr allocated.
783 * TBD: shared memory region support
786 __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
787 u32 *stag, u8 stag_state, u32 pdid,
788 enum tpt_mem_type type, enum tpt_mem_perm perm,
789 u32 zbva, u64 to, u32 len, u8 page_size, __be64 *pbl,
790 u32 *pbl_size, u32 *pbl_addr)
793 struct tpt_entry tpt;
796 int rereg = (*stag != T3_STAG_UNSET);
798 stag_state = stag_state > 0;
799 stag_idx = (*stag) >> 8;
801 if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) {
802 stag_idx = cxio_hal_get_stag(rdev_p->rscp);
805 *stag = (stag_idx << 8) | ((*stag) & 0xFF);
807 CTR5(KTR_IW_CXGB, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x",
808 __FUNCTION__, stag_state, type, pdid, stag_idx);
811 cxio_hal_pblpool_free(rdev_p, *pbl_addr, *pbl_size << 3);
813 *pbl_addr = cxio_hal_pblpool_alloc(rdev_p, *pbl_size << 3);
819 mtx_lock(&rdev_p->ctrl_qp.lock);
821 /* write PBL first if any - update pbl only if pbl list exist */
824 CTR4(KTR_IW_CXGB, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d",
825 __FUNCTION__, *pbl_addr, rdev_p->rnic_info.pbl_base,
827 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
829 (*pbl_size << 3), pbl, 0);
834 /* write TPT entry */
836 memset(&tpt, 0, sizeof(tpt));
838 tpt.valid_stag_pdid = htobe32(F_TPT_VALID |
839 V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) |
840 V_TPT_STAG_STATE(stag_state) |
841 V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));
842 PANIC_IF(page_size >= 28);
843 tpt.flags_pagesize_qpid = htobe32(V_TPT_PERM(perm) |
844 F_TPT_MW_BIND_ENABLE |
845 V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
846 V_TPT_PAGE_SIZE(page_size));
847 tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
848 htobe32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, *pbl_addr)>>3));
849 tpt.len = htobe32(len);
850 tpt.va_hi = htobe32((u32) (to >> 32));
851 tpt.va_low_or_fbo = htobe32((u32) (to & 0xFFFFFFFFULL));
852 tpt.rsvd_bind_cnt_or_pstag = 0;
853 tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
854 htobe32(V_TPT_PBL_SIZE((*pbl_size) >> 2));
856 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
858 (rdev_p->rnic_info.tpt_base >> 5),
859 sizeof(tpt), &tpt, 1);
861 /* release the stag index to free pool */
863 cxio_hal_put_stag(rdev_p->rscp, stag_idx);
865 wptr = rdev_p->ctrl_qp.wptr;
866 mtx_unlock(&rdev_p->ctrl_qp.lock);
868 if (cxio_wait(&rdev_p->ctrl_qp,
869 &rdev_p->ctrl_qp.lock,
870 SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr)))
876 cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
877 enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
878 u8 page_size, __be64 *pbl, u32 *pbl_size,
881 *stag = T3_STAG_UNSET;
882 return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
883 zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
887 cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
888 enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
889 u8 page_size, __be64 *pbl, u32 *pbl_size,
892 return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
893 zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
897 cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
900 return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
901 &pbl_size, &pbl_addr);
905 cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
908 *stag = T3_STAG_UNSET;
909 return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
910 NULL, &pbl_size, NULL);
914 cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
916 return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
921 cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
923 struct t3_rdma_init_wr *wqe;
924 struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
927 CTR2(KTR_IW_CXGB, "%s rdev_p %p", __FUNCTION__, rdev_p);
928 wqe = mtod(m, struct t3_rdma_init_wr *);
929 m->m_len = m->m_pkthdr.len = sizeof(*wqe);
930 wqe->wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_INIT));
931 wqe->wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(attr->tid) |
932 V_FW_RIWR_LEN(sizeof(*wqe) >> 3));
934 wqe->qpid = htobe32(attr->qpid);
935 wqe->pdid = htobe32(attr->pdid);
936 wqe->scqid = htobe32(attr->scqid);
937 wqe->rcqid = htobe32(attr->rcqid);
938 wqe->rq_addr = htobe32(attr->rq_addr - rdev_p->rnic_info.rqt_base);
939 wqe->rq_size = htobe32(attr->rq_size);
940 wqe->mpaattrs = attr->mpaattrs;
941 wqe->qpcaps = attr->qpcaps;
942 wqe->ulpdu_size = htobe16(attr->tcp_emss);
943 wqe->flags = htobe32(attr->flags);
944 wqe->ord = htobe32(attr->ord);
945 wqe->ird = htobe32(attr->ird);
946 wqe->qp_dma_addr = htobe64(attr->qp_dma_addr);
947 wqe->qp_dma_size = htobe32(attr->qp_dma_size);
948 wqe->irs = htobe32(attr->irs);
949 m_set_priority(m, 0); /* 0=>ToeQ; 1=>CtrlQ */
952 return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
956 cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
962 cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
968 cxio_hal_ev_handler(struct t3cdev *t3cdev_p, struct mbuf *m)
971 struct cxio_rdev *rdev_p = NULL;
972 struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data;
974 CTR6(KTR_IW_CXGB, "%s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x",
975 __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
976 RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg));
977 CTR4(KTR_IW_CXGB, "se %0x notify %0x cqbranch %0x creditth %0x",
978 RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
979 RSPQ_CREDIT_THRESH(rsp_msg));
980 CTR4(KTR_IW_CXGB, "CQE: QPID 0x%0x type 0x%0x status 0x%0x opcode %d",
981 CQE_QPID(rsp_msg->cqe),
982 CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
983 CQE_OPCODE(rsp_msg->cqe));
984 CTR3(KTR_IW_CXGB, "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
985 CQE_LEN(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
986 rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;
988 CTR2(KTR_IW_CXGB, "%s called by t3cdev %p with null ulp", __FUNCTION__,
992 if (CQE_QPID(rsp_msg->cqe) == T3_CTRL_QP_ID) {
993 mtx_lock(&rdev_p->ctrl_qp.lock);
994 rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
995 wakeup(&rdev_p->ctrl_qp);
996 mtx_unlock(&rdev_p->ctrl_qp.lock);
998 } else if (CQE_QPID(rsp_msg->cqe) == 0xfff8)
1000 else if (cxio_ev_cb)
1001 (*cxio_ev_cb) (rdev_p, m);
1008 /* Caller takes care of locking if needed */
1010 cxio_rdev_open(struct cxio_rdev *rdev_p)
1015 if (strlen(rdev_p->dev_name)) {
1016 if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {
1023 } else if (rdev_p->t3cdev_p) {
1024 if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p))
1026 ifp = rdev_p->t3cdev_p->lldev;
1027 strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,
1028 T3_MAX_DEV_NAME_LEN);
1030 CTR1(KTR_IW_CXGB, "%s t3cdev_p or dev_name must be set", __FUNCTION__);
1034 TAILQ_INSERT_TAIL(&rdev_list, rdev_p, entry);
1036 CTR2(KTR_IW_CXGB, "%s opening rnic dev %s", __FUNCTION__, rdev_p->dev_name);
1037 memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
1038 if (!rdev_p->t3cdev_p)
1039 rdev_p->t3cdev_p = T3CDEV(ifp);
1040 rdev_p->t3cdev_p->ulp = (void *) rdev_p;
1041 err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,
1042 &(rdev_p->rnic_info));
1044 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1045 __FUNCTION__, rdev_p->t3cdev_p, err);
1048 err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,
1049 &(rdev_p->port_info));
1051 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1052 __FUNCTION__, rdev_p->t3cdev_p, err);
1057 * qpshift is the number of bits to shift the qpid left in order
1058 * to get the correct address of the doorbell for that qp.
1060 cxio_init_ucontext(rdev_p, &rdev_p->uctx);
1061 rdev_p->qpshift = PAGE_SHIFT -
1063 ilog2(rdev_p->rnic_info.udbell_len >>
1065 rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
1066 rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
1067 CTR4(KTR_IW_CXGB, "cxio_rdev_open rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d",
1068 rdev_p->dev_name, rdev_p->rnic_info.tpt_base,
1069 rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p));
1070 CTR4(KTR_IW_CXGB, "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x",
1071 rdev_p->rnic_info.pbl_base,
1072 rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
1073 rdev_p->rnic_info.rqt_top);
1074 CTR6(KTR_IW_CXGB, "udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
1075 "qpnr %d qpmask 0x%x",
1076 rdev_p->rnic_info.udbell_len,
1077 rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
1078 rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);
1080 err = cxio_hal_init_ctrl_qp(rdev_p);
1082 log(LOG_ERR, "%s error %d initializing ctrl_qp.\n",
1086 err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
1087 0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,
1090 log(LOG_ERR, "%s error %d initializing hal resources.\n",
1094 err = cxio_hal_pblpool_create(rdev_p);
1096 log(LOG_ERR, "%s error %d initializing pbl mem pool.\n",
1100 err = cxio_hal_rqtpool_create(rdev_p);
1102 log(LOG_ERR, "%s error %d initializing rqt mem pool.\n",
1108 cxio_hal_pblpool_destroy(rdev_p);
1110 cxio_hal_destroy_resource(rdev_p->rscp);
1112 cxio_hal_destroy_ctrl_qp(rdev_p);
1114 TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1119 cxio_rdev_close(struct cxio_rdev *rdev_p)
1122 cxio_hal_pblpool_destroy(rdev_p);
1123 cxio_hal_rqtpool_destroy(rdev_p);
1124 TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1125 rdev_p->t3cdev_p->ulp = NULL;
1126 cxio_hal_destroy_ctrl_qp(rdev_p);
1127 cxio_hal_destroy_resource(rdev_p->rscp);
1134 TAILQ_INIT(&rdev_list);
1136 if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))
1139 t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
1146 struct cxio_rdev *rdev, *tmp;
1148 t3_register_cpl_handler(CPL_ASYNC_NOTIF, NULL);
1149 TAILQ_FOREACH_SAFE(rdev, &rdev_list, entry, tmp)
1150 cxio_rdev_close(rdev);
1152 cxio_hal_destroy_rhdl_resource();
1157 flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
1159 struct t3_swsq *sqp;
1160 __u32 ptr = wq->sq_rptr;
1161 int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
1163 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1165 if (!sqp->signaled) {
1167 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1168 } else if (sqp->complete) {
1171 * Insert this completed cqe into the swcq.
1173 CTR3(KTR_IW_CXGB, "%s moving cqe into swcq sq idx %ld cq idx %ld",
1174 __FUNCTION__, Q_PTR2IDX(ptr, wq->sq_size_log2),
1175 Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
1176 sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
1177 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
1187 create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
1188 struct t3_cqe *read_cqe)
1190 read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
1191 read_cqe->len = wq->oldest_read->read_len;
1192 read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
1193 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
1194 V_CQE_OPCODE(T3_READ_REQ) |
1199 * Return a ptr to the next read wr in the SWSQ or NULL.
1202 advance_oldest_read(struct t3_wq *wq)
1205 u32 rptr = wq->oldest_read - wq->sq + 1;
1206 u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
1208 while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
1209 wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
1211 if (wq->oldest_read->opcode == T3_READ_REQ)
1215 wq->oldest_read = NULL;
1222 * check the validity of the first CQE,
1223 * supply the wq assicated with the qpid.
1225 * credit: cq credit to return to sge.
1226 * cqe_flushed: 1 iff the CQE is flushed.
1227 * cqe: copy of the polled CQE.
1231 * -1 CQE skipped, try again.
1234 cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
1235 u8 *cqe_flushed, u64 *cookie, u32 *credit)
1238 struct t3_cqe *hw_cqe, read_cqe;
1242 hw_cqe = cxio_next_cqe(cq);
1244 CTR5(KTR_IW_CXGB, "cxio_poll_cq CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x",
1245 CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
1246 CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe));
1247 CTR4(KTR_IW_CXGB, "opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
1248 CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
1249 CQE_WRID_LOW(*hw_cqe));
1252 * skip cqe's not affiliated with a QP.
1260 * Gotta tweak READ completions:
1261 * 1) the cqe doesn't contain the sq_wptr from the wr.
1262 * 2) opcode not reflected from the wr.
1263 * 3) read_len not reflected from the wr.
1264 * 4) cq_type is RQ_TYPE not SQ_TYPE.
1266 if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
1269 * Don't write to the HWCQ, so create a new read req CQE
1272 create_read_req_cqe(wq, hw_cqe, &read_cqe);
1274 advance_oldest_read(wq);
1278 * T3A: Discard TERMINATE CQEs.
1280 if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
1286 if (CQE_STATUS(*hw_cqe) || wq->error) {
1287 *cqe_flushed = wq->error;
1291 * T3A inserts errors into the CQE. We cannot return
1292 * these as work completions.
1294 /* incoming write failures */
1295 if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
1296 && RQ_TYPE(*hw_cqe)) {
1300 /* incoming read request failures */
1301 if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
1306 /* incoming SEND with no receive posted failures */
1307 if ((CQE_OPCODE(*hw_cqe) == T3_SEND) && RQ_TYPE(*hw_cqe) &&
1308 Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1318 if (RQ_TYPE(*hw_cqe)) {
1321 * HW only validates 4 bits of MSN. So we must validate that
1322 * the MSN in the SEND is the next expected MSN. If its not,
1323 * then we complete this with TPT_ERR_MSN and mark the wq in
1326 if (__predict_false((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
1328 hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
1335 * If we get here its a send completion.
1337 * Handle out of order completion. These get stuffed
1338 * in the SW SQ. Then the SW SQ is walked to move any
1339 * now in-order completions into the SW CQ. This handles
1341 * 1) reaping unsignaled WRs when the first subsequent
1342 * signaled WR is completed.
1343 * 2) out of order read completions.
1345 if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
1346 struct t3_swsq *sqp;
1348 CTR2(KTR_IW_CXGB, "%s out of order completion going in swsq at idx %ld",
1350 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
1352 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
1363 * Reap the associated WR(s) that are freed up with this
1366 if (SQ_TYPE(*hw_cqe)) {
1367 wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
1368 CTR2(KTR_IW_CXGB, "%s completing sq idx %ld", __FUNCTION__,
1369 Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
1371 Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
1374 CTR2(KTR_IW_CXGB, "%s completing rq idx %ld", __FUNCTION__,
1375 Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1376 *cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1382 * Flush any completed cqes that are now in-order.
1384 flush_completed_wrs(wq, cq);
1387 if (SW_CQE(*hw_cqe)) {
1388 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x",
1389 __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
1392 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip hw cqe rptr 0x%x",
1393 __FUNCTION__, cq, cq->cqid, cq->rptr);
1397 * T3A: compute credits.
1399 if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
1400 || ((cq->rptr - cq->wptr) >= 128)) {
1401 *credit = cq->rptr - cq->wptr;
1402 cq->wptr = cq->rptr;