2 /**************************************************************************
4 Copyright (c) 2007, Chelsio Inc.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
29 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/pciio.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/bus_dma.h>
44 #include <sys/ioccom.h>
46 #include <sys/rwlock.h>
47 #include <sys/linker.h>
48 #include <sys/firmware.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/queue.h>
55 #include <sys/taskqueue.h>
57 #include <sys/queue.h>
62 #include <netinet/in.h>
64 #include <contrib/rdma/ib_verbs.h>
68 #include <cxgb_include.h>
69 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
70 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
71 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
72 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
73 #include <ulp/iw_cxgb/iw_cxgb.h>
74 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
76 #include <dev/cxgb/cxgb_include.h>
77 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_wr.h>
78 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.h>
79 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.h>
80 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h>
81 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb.h>
82 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.h>
85 static TAILQ_HEAD( ,cxio_rdev) rdev_list;
86 static cxio_hal_ev_callback_func_t cxio_ev_cb = NULL;
88 static struct cxio_rdev *
89 cxio_hal_find_rdev_by_name(char *dev_name)
91 struct cxio_rdev *rdev;
93 TAILQ_FOREACH(rdev, &rdev_list, entry)
94 if (!strcmp(rdev->dev_name, dev_name))
100 cxio_hal_find_rdev_by_t3cdev(struct t3cdev *tdev)
102 struct cxio_rdev *rdev;
104 TAILQ_FOREACH(rdev, &rdev_list, entry)
105 if (rdev->t3cdev_p == tdev)
111 cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,
112 enum t3_cq_opcode op, u32 credit)
118 struct rdma_cq_op setup;
120 setup.credits = (op == CQ_CREDIT_UPDATE) ? credit : 0;
122 ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);
124 if ((ret < 0) || (op == CQ_CREDIT_UPDATE))
128 * If the rearm returned an index other than our current index,
129 * then there might be CQE's in flight (being DMA'd). We must wait
130 * here for them to complete or the consumer can miss a notification.
132 if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {
138 * Keep the generation correct by bumping rptr until it
139 * matches the index returned by the rearm - 1.
141 while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)
145 * Now rptr is the index for the (last) cqe that was
146 * in-flight at the time the HW rearmed the CQ. We
147 * spin until that CQE is valid.
149 cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);
150 while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {
154 log(LOG_ERR, "%s: stalled rnic\n",
167 cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)
169 struct rdma_cq_setup setup;
171 setup.base_addr = 0; /* NULL address */
172 setup.size = 0; /* disaable the CQ */
174 setup.credit_thres = 0;
176 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
180 cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
183 struct t3_modify_qp_wr *wqe;
184 struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
186 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
189 wqe = mtod(m, struct t3_modify_qp_wr *);
190 m->m_len = m->m_pkthdr.len = sizeof(*wqe);
191 memset(wqe, 0, sizeof(*wqe));
192 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);
193 wqe->flags = htobe32(MODQP_WRITE_EC);
194 sge_cmd = qpid << 8 | 3;
195 wqe->sge_cmd = htobe64(sge_cmd);
196 m_set_priority(m, CPL_PRIORITY_CONTROL);
199 return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
203 cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
205 struct rdma_cq_setup setup;
206 int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);
208 cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);
211 cq->sw_queue = malloc(size, M_DEVBUF, M_NOWAIT|M_ZERO);
215 cq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
216 (1UL << (cq->size_log2)) *
217 sizeof(struct t3_cqe),
218 &(cq->dma_addr), M_NOWAIT);
220 cq->queue = contigmalloc((1UL << (cq->size_log2))*sizeof(struct t3_cqe),
221 M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
223 cq->dma_addr = vtophys(cq->queue);
225 free(cq->sw_queue, M_DEVBUF);
231 pci_unmap_addr_set(cq, mapping, cq->dma_addr);
233 memset(cq->queue, 0, size);
235 setup.base_addr = (u64) (cq->dma_addr);
236 setup.size = 1UL << cq->size_log2;
237 setup.credits = 65535;
238 setup.credit_thres = 1;
239 if (rdev_p->t3cdev_p->type != T3A)
243 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
247 cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
249 struct rdma_cq_setup setup;
251 setup.base_addr = (u64) (cq->dma_addr);
252 setup.size = 1UL << cq->size_log2;
253 setup.credits = setup.size;
254 setup.credit_thres = setup.size; /* TBD: overflow recovery */
256 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
260 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
262 struct cxio_qpid *entry;
266 mtx_lock(&uctx->lock);
267 if (!TAILQ_EMPTY(&uctx->qpids)) {
269 entry = TAILQ_FIRST(&uctx->qpids);
270 TAILQ_REMOVE(&uctx->qpids, entry, entry);
272 free(entry, M_DEVBUF);
274 qpid = cxio_hal_get_qpid(rdev_p->rscp);
277 for (i = qpid+1; i & rdev_p->qpmask; i++) {
278 entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
282 TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
286 mtx_unlock(&uctx->lock);
287 CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
292 put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
293 struct cxio_ucontext *uctx)
295 struct cxio_qpid *entry;
297 entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
298 CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
300 mtx_lock(&uctx->lock);
301 TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
302 mtx_unlock(&uctx->lock);
306 cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
308 struct cxio_qpid *pos, *tmp;
310 mtx_lock(&uctx->lock);
311 TAILQ_FOREACH_SAFE(pos, &uctx->qpids, entry, tmp) {
312 TAILQ_REMOVE(&uctx->qpids, pos, entry);
313 if (!(pos->qpid & rdev_p->qpmask))
314 cxio_hal_put_qpid(rdev_p->rscp, pos->qpid);
317 mtx_unlock(&uctx->lock);
321 cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
323 TAILQ_INIT(&uctx->qpids);
324 mtx_init(&uctx->lock, "cxio uctx", NULL, MTX_DEF|MTX_DUPOK);
328 cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
329 struct t3_wq *wq, struct cxio_ucontext *uctx)
331 int depth = 1UL << wq->size_log2;
332 int rqsize = 1UL << wq->rq_size_log2;
334 wq->qpid = get_qpid(rdev_p, uctx);
338 wq->rq = malloc(depth * sizeof(u64), M_DEVBUF, M_NOWAIT|M_ZERO);
342 wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);
346 wq->sq = malloc(depth * sizeof(struct t3_swsq), M_DEVBUF, M_NOWAIT|M_ZERO);
350 wq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
351 depth * sizeof(union t3_wr),
352 &(wq->dma_addr), M_NOWAIT);
354 wq->queue = contigmalloc(depth *sizeof(union t3_wr),
355 M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
357 wq->dma_addr = vtophys(wq->queue);
363 memset(wq->queue, 0, depth * sizeof(union t3_wr));
365 pci_unmap_addr_set(wq, mapping, wq->dma_addr);
367 wq->doorbell = rdev_p->rnic_info.kdb_addr;
369 wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
370 (wq->qpid << rdev_p->qpshift);
371 CTR4(KTR_IW_CXGB, "%s qpid 0x%x doorbell 0x%p udb 0x%llx", __FUNCTION__,
372 wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
375 free(wq->sq, M_DEVBUF);
377 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);
379 free(wq->rq, M_DEVBUF);
381 put_qpid(rdev_p, wq->qpid, uctx);
386 cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
389 err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
390 free(cq->sw_queue, M_DEVBUF);
392 dma_free_coherent(&(rdev_p->rnic_info.pdev),
393 (1UL << (cq->size_log2))
394 * sizeof(struct t3_cqe), cq->queue,
395 /* pci_unmap_addr(cq, mapping)*/ 0);
397 contigfree(cq->queue,(1UL << (cq->size_log2))
398 * sizeof(struct t3_cqe), M_DEVBUF);
400 cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
405 cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
406 struct cxio_ucontext *uctx)
410 dma_free_coherent(&(rdev_p->rnic_info.pdev),
411 (1UL << (wq->size_log2))
412 * sizeof(union t3_wr), wq->queue,
413 /* pci_unmap_addr(wq, mapping)*/ 0);
415 contigfree(wq->queue, (1UL << (wq->size_log2))
416 * sizeof(union t3_wr), M_DEVBUF);
418 free(wq->sq, M_DEVBUF);
419 cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));
420 free(wq->rq, M_DEVBUF);
421 put_qpid(rdev_p, wq->qpid, uctx);
426 insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
430 CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
431 wq, cq, cq->sw_rptr, cq->sw_wptr);
432 memset(&cqe, 0, sizeof(cqe));
433 cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
434 V_CQE_OPCODE(T3_SEND) |
437 V_CQE_QPID(wq->qpid) |
438 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
440 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
445 cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
449 CTR3(KTR_IW_CXGB, "%s wq %p cq %p", __FUNCTION__, wq, cq);
452 CTR4(KTR_IW_CXGB, "%s rq_rptr %u rq_wptr %u skip count %u", __FUNCTION__,
453 wq->rq_rptr, wq->rq_wptr, count);
454 ptr = wq->rq_rptr + count;
455 while (ptr++ != wq->rq_wptr)
456 insert_recv_cqe(wq, cq);
460 insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
465 CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
466 wq, cq, cq->sw_rptr, cq->sw_wptr);
467 memset(&cqe, 0, sizeof(cqe));
468 cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
469 V_CQE_OPCODE(sqp->opcode) |
472 V_CQE_QPID(wq->qpid) |
473 V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
475 cqe.u.scqe.wrid_hi = sqp->sq_wptr;
477 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
482 cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
485 struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
487 ptr = wq->sq_rptr + count;
489 while (ptr != wq->sq_wptr) {
490 insert_sq_cqe(wq, cq, sqp);
497 * Move all CQEs from the HWCQ into the SWCQ.
500 cxio_flush_hw_cq(struct t3_cq *cq)
502 struct t3_cqe *cqe, *swcqe;
504 CTR3(KTR_IW_CXGB, "%s cq %p cqid 0x%x", __FUNCTION__, cq, cq->cqid);
505 cqe = cxio_next_hw_cqe(cq);
507 CTR3(KTR_IW_CXGB, "%s flushing hwcq rptr 0x%x to swcq wptr 0x%x",
508 __FUNCTION__, cq->rptr, cq->sw_wptr);
509 swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
511 swcqe->header |= htobe32(V_CQE_SWCQE(1));
514 cqe = cxio_next_hw_cqe(cq);
518 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)
520 if (CQE_OPCODE(*cqe) == T3_TERMINATE)
523 if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe))
526 if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe))
529 if ((CQE_OPCODE(*cqe) == T3_SEND) && RQ_TYPE(*cqe) &&
530 Q_EMPTY(wq->rq_rptr, wq->rq_wptr))
537 cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
544 while (!Q_EMPTY(ptr, cq->sw_wptr)) {
545 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
546 if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) &&
547 (CQE_QPID(*cqe) == wq->qpid))
551 CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
555 cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
561 CTR2(KTR_IW_CXGB, "%s count zero %d", __FUNCTION__, *count);
563 while (!Q_EMPTY(ptr, cq->sw_wptr)) {
564 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
565 if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) &&
566 (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq))
570 CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
574 cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)
576 struct rdma_cq_setup setup;
578 setup.base_addr = 0; /* NULL address */
579 setup.size = 1; /* enable the CQ */
582 /* force SGE to redirect to RspQ and interrupt */
583 setup.credit_thres = 0;
585 return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
589 cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
592 u64 sge_cmd, ctx0, ctx1;
594 struct t3_modify_qp_wr *wqe;
597 m = m_gethdr(MT_DATA, M_NOWAIT);
599 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
602 err = cxio_hal_init_ctrl_cq(rdev_p);
604 CTR2(KTR_IW_CXGB, "%s err %d initializing ctrl_cq", __FUNCTION__, err);
608 rdev_p->ctrl_qp.workq = dma_alloc_coherent(
609 rdev_p->rnic_info.pdev,
610 (1 << T3_CTRL_QP_SIZE_LOG2) *
612 &(rdev_p->ctrl_qp.dma_addr),
615 rdev_p->ctrl_qp.workq = contigmalloc((1 << T3_CTRL_QP_SIZE_LOG2)
616 *sizeof(union t3_wr), M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
617 if (rdev_p->ctrl_qp.workq)
618 rdev_p->ctrl_qp.dma_addr = vtophys(rdev_p->ctrl_qp.workq);
622 if (!rdev_p->ctrl_qp.workq) {
623 CTR1(KTR_IW_CXGB, "%s dma_alloc_coherent failed", __FUNCTION__);
628 pci_unmap_addr_set(&rdev_p->ctrl_qp, mapping,
629 rdev_p->ctrl_qp.dma_addr);
631 rdev_p->ctrl_qp.doorbell = (void /*__iomem */ *)rdev_p->rnic_info.kdb_addr;
632 memset(rdev_p->ctrl_qp.workq, 0,
633 (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));
635 mtx_init(&rdev_p->ctrl_qp.lock, "ctl-qp lock", NULL, MTX_DEF|MTX_DUPOK);
637 /* update HW Ctrl QP context */
638 base_addr = rdev_p->ctrl_qp.dma_addr;
640 ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) |
641 V_EC_BASE_LO((u32) base_addr & 0xffff));
643 ctx0 |= V_EC_CREDITS(FW_WR_NUM);
645 ctx1 = (u32) base_addr;
647 ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) |
648 V_EC_TYPE(0) | V_EC_GEN(1) |
649 V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) << 32;
650 wqe = mtod(m, struct t3_modify_qp_wr *);
651 m->m_len = m->m_pkthdr.len = sizeof(*wqe);
652 memset(wqe, 0, sizeof(*wqe));
653 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0,
655 wqe->flags = htobe32(MODQP_WRITE_EC);
656 sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
657 wqe->sge_cmd = htobe64(sge_cmd);
658 wqe->ctx1 = htobe64(ctx1);
659 wqe->ctx0 = htobe64(ctx0);
660 CTR3(KTR_IW_CXGB, "CtrlQP dma_addr 0x%llx workq %p size %d",
661 (unsigned long long) rdev_p->ctrl_qp.dma_addr,
662 rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
663 m_set_priority(m, CPL_PRIORITY_CONTROL);
666 return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
673 cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
677 dma_free_coherent(&(rdev_p->rnic_info.pdev),
678 (1UL << T3_CTRL_QP_SIZE_LOG2)
679 * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,
680 /* pci_unmap_addr(&rdev_p->ctrl_qp, mapping)*/ 0);
682 contigfree(rdev_p->ctrl_qp.workq,(1UL << T3_CTRL_QP_SIZE_LOG2)
683 * sizeof(union t3_wr), M_DEVBUF);
685 return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);
688 /* write len bytes of data into addr (32B aligned address)
689 * If data is NULL, clear len byte of memory to zero.
690 * caller aquires the ctrl_qp lock before the call
693 cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
694 u32 len, void *data, int completion)
696 u32 i, nr_wqe, copy_len;
698 u8 wr_len, utx_len; /* lenght in 8 byte flit */
699 enum t3_wr_flags flag;
703 nr_wqe = len % 96 ? len / 96 + 1 : len / 96; /* 96B max per WQE */
704 CTR6(KTR_IW_CXGB, "cxio_hal_ctrl_qp_write_mem wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x",
705 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,
707 utx_len = 3; /* in 32B unit */
708 for (i = 0; i < nr_wqe; i++) {
709 if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,
710 T3_CTRL_QP_SIZE_LOG2)) {
711 CTR4(KTR_IW_CXGB, "%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, "
712 "wait for more space i %d", __FUNCTION__,
713 rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);
714 if (cxio_wait(&rdev_p->ctrl_qp,
715 &rdev_p->ctrl_qp.lock,
716 !Q_FULL(rdev_p->ctrl_qp.rptr,
717 rdev_p->ctrl_qp.wptr,
718 T3_CTRL_QP_SIZE_LOG2))) {
719 CTR1(KTR_IW_CXGB, "%s ctrl_qp workq interrupted",
723 CTR2(KTR_IW_CXGB, "%s ctrl_qp wakeup, continue posting work request "
724 "i %d", __FUNCTION__, i);
726 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
727 (1 << T3_CTRL_QP_SIZE_LOG2)));
729 if (i == (nr_wqe - 1)) {
731 flag = completion ? T3_COMPLETION_FLAG : 0;
733 utx_len = len / 32 + 1;
739 * Force a CQE to return the credit to the workq in case
740 * we posted more than half the max QP size of WRs
743 (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) {
744 flag = T3_COMPLETION_FLAG;
745 CTR2(KTR_IW_CXGB, "%s force completion at i %d", __FUNCTION__, i);
748 /* build the utx mem command */
749 wqe += (sizeof(struct t3_bypass_wr) >> 3);
750 utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3);
752 utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1);
753 *wqe = htobe64(utx_cmd);
755 copy_data = (u8 *) data + i * 96;
756 copy_len = len > 96 ? 96 : len;
758 /* clear memory content if data is NULL */
760 memcpy(wqe, copy_data, copy_len);
762 memset(wqe, 0, copy_len);
764 memset(((u8 *) wqe) + copy_len, 0,
765 32 - (copy_len % 32));
766 wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 +
768 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
769 (1 << T3_CTRL_QP_SIZE_LOG2)));
771 /* wptr in the WRID[31:0] */
772 ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;
775 * This must be the last write with a memory barrier
778 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
779 Q_GENBIT(rdev_p->ctrl_qp.wptr,
780 T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,
782 if (flag == T3_COMPLETION_FLAG)
783 ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
786 rdev_p->ctrl_qp.wptr++;
791 /* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size
792 * OUT: stag index, actual pbl_size, pbl_addr allocated.
793 * TBD: shared memory region support
796 __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
797 u32 *stag, u8 stag_state, u32 pdid,
798 enum tpt_mem_type type, enum tpt_mem_perm perm,
799 u32 zbva, u64 to, u32 len, u8 page_size, __be64 *pbl,
800 u32 *pbl_size, u32 *pbl_addr)
803 struct tpt_entry tpt;
806 int rereg = (*stag != T3_STAG_UNSET);
808 stag_state = stag_state > 0;
809 stag_idx = (*stag) >> 8;
811 if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) {
812 stag_idx = cxio_hal_get_stag(rdev_p->rscp);
815 *stag = (stag_idx << 8) | ((*stag) & 0xFF);
817 CTR5(KTR_IW_CXGB, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x",
818 __FUNCTION__, stag_state, type, pdid, stag_idx);
821 cxio_hal_pblpool_free(rdev_p, *pbl_addr, *pbl_size << 3);
823 *pbl_addr = cxio_hal_pblpool_alloc(rdev_p, *pbl_size << 3);
829 mtx_lock(&rdev_p->ctrl_qp.lock);
831 /* write PBL first if any - update pbl only if pbl list exist */
834 CTR4(KTR_IW_CXGB, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d",
835 __FUNCTION__, *pbl_addr, rdev_p->rnic_info.pbl_base,
837 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
839 (*pbl_size << 3), pbl, 0);
844 /* write TPT entry */
846 memset(&tpt, 0, sizeof(tpt));
848 tpt.valid_stag_pdid = htobe32(F_TPT_VALID |
849 V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) |
850 V_TPT_STAG_STATE(stag_state) |
851 V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));
852 PANIC_IF(page_size >= 28);
853 tpt.flags_pagesize_qpid = htobe32(V_TPT_PERM(perm) |
854 F_TPT_MW_BIND_ENABLE |
855 V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
856 V_TPT_PAGE_SIZE(page_size));
857 tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
858 htobe32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, *pbl_addr)>>3));
859 tpt.len = htobe32(len);
860 tpt.va_hi = htobe32((u32) (to >> 32));
861 tpt.va_low_or_fbo = htobe32((u32) (to & 0xFFFFFFFFULL));
862 tpt.rsvd_bind_cnt_or_pstag = 0;
863 tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
864 htobe32(V_TPT_PBL_SIZE((*pbl_size) >> 2));
866 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
868 (rdev_p->rnic_info.tpt_base >> 5),
869 sizeof(tpt), &tpt, 1);
871 /* release the stag index to free pool */
873 cxio_hal_put_stag(rdev_p->rscp, stag_idx);
875 wptr = rdev_p->ctrl_qp.wptr;
876 mtx_unlock(&rdev_p->ctrl_qp.lock);
878 if (cxio_wait(&rdev_p->ctrl_qp,
879 &rdev_p->ctrl_qp.lock,
880 SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr)))
886 cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
887 enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
888 u8 page_size, __be64 *pbl, u32 *pbl_size,
891 *stag = T3_STAG_UNSET;
892 return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
893 zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
897 cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
898 enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
899 u8 page_size, __be64 *pbl, u32 *pbl_size,
902 return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
903 zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
907 cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
910 return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
911 &pbl_size, &pbl_addr);
915 cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
918 *stag = T3_STAG_UNSET;
919 return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
920 NULL, &pbl_size, NULL);
924 cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
926 return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
931 cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
933 struct t3_rdma_init_wr *wqe;
934 struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
937 CTR2(KTR_IW_CXGB, "%s rdev_p %p", __FUNCTION__, rdev_p);
938 wqe = mtod(m, struct t3_rdma_init_wr *);
939 m->m_len = m->m_pkthdr.len = sizeof(*wqe);
940 wqe->wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_INIT));
941 wqe->wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(attr->tid) |
942 V_FW_RIWR_LEN(sizeof(*wqe) >> 3));
944 wqe->qpid = htobe32(attr->qpid);
945 wqe->pdid = htobe32(attr->pdid);
946 wqe->scqid = htobe32(attr->scqid);
947 wqe->rcqid = htobe32(attr->rcqid);
948 wqe->rq_addr = htobe32(attr->rq_addr - rdev_p->rnic_info.rqt_base);
949 wqe->rq_size = htobe32(attr->rq_size);
950 wqe->mpaattrs = attr->mpaattrs;
951 wqe->qpcaps = attr->qpcaps;
952 wqe->ulpdu_size = htobe16(attr->tcp_emss);
953 wqe->flags = htobe32(attr->flags);
954 wqe->ord = htobe32(attr->ord);
955 wqe->ird = htobe32(attr->ird);
956 wqe->qp_dma_addr = htobe64(attr->qp_dma_addr);
957 wqe->qp_dma_size = htobe32(attr->qp_dma_size);
958 wqe->irs = htobe32(attr->irs);
959 m_set_priority(m, 0); /* 0=>ToeQ; 1=>CtrlQ */
962 return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
966 cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
972 cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
978 cxio_hal_ev_handler(struct t3cdev *t3cdev_p, struct mbuf *m)
981 struct cxio_rdev *rdev_p = NULL;
982 struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data;
984 CTR6(KTR_IW_CXGB, "%s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x",
985 __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
986 RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg));
987 CTR4(KTR_IW_CXGB, "se %0x notify %0x cqbranch %0x creditth %0x",
988 RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
989 RSPQ_CREDIT_THRESH(rsp_msg));
990 CTR4(KTR_IW_CXGB, "CQE: QPID 0x%0x type 0x%0x status 0x%0x opcode %d",
991 CQE_QPID(rsp_msg->cqe),
992 CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
993 CQE_OPCODE(rsp_msg->cqe));
994 CTR3(KTR_IW_CXGB, "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
995 CQE_LEN(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
996 rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;
998 CTR2(KTR_IW_CXGB, "%s called by t3cdev %p with null ulp", __FUNCTION__,
1002 if (CQE_QPID(rsp_msg->cqe) == T3_CTRL_QP_ID) {
1003 mtx_lock(&rdev_p->ctrl_qp.lock);
1004 rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
1005 wakeup(&rdev_p->ctrl_qp);
1006 mtx_unlock(&rdev_p->ctrl_qp.lock);
1008 } else if (CQE_QPID(rsp_msg->cqe) == 0xfff8)
1010 else if (cxio_ev_cb)
1011 (*cxio_ev_cb) (rdev_p, m);
1018 /* Caller takes care of locking if needed */
1020 cxio_rdev_open(struct cxio_rdev *rdev_p)
1025 if (strlen(rdev_p->dev_name)) {
1026 if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {
1033 } else if (rdev_p->t3cdev_p) {
1034 if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p))
1036 ifp = rdev_p->t3cdev_p->lldev;
1037 strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,
1038 T3_MAX_DEV_NAME_LEN);
1040 CTR1(KTR_IW_CXGB, "%s t3cdev_p or dev_name must be set", __FUNCTION__);
1044 TAILQ_INSERT_TAIL(&rdev_list, rdev_p, entry);
1046 CTR2(KTR_IW_CXGB, "%s opening rnic dev %s", __FUNCTION__, rdev_p->dev_name);
1047 memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
1048 if (!rdev_p->t3cdev_p)
1049 rdev_p->t3cdev_p = T3CDEV(ifp);
1050 rdev_p->t3cdev_p->ulp = (void *) rdev_p;
1051 err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,
1052 &(rdev_p->rnic_info));
1054 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1055 __FUNCTION__, rdev_p->t3cdev_p, err);
1058 err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,
1059 &(rdev_p->port_info));
1061 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1062 __FUNCTION__, rdev_p->t3cdev_p, err);
1067 * qpshift is the number of bits to shift the qpid left in order
1068 * to get the correct address of the doorbell for that qp.
1070 cxio_init_ucontext(rdev_p, &rdev_p->uctx);
1071 rdev_p->qpshift = PAGE_SHIFT -
1073 ilog2(rdev_p->rnic_info.udbell_len >>
1075 rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
1076 rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
1077 CTR4(KTR_IW_CXGB, "cxio_rdev_open rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d",
1078 rdev_p->dev_name, rdev_p->rnic_info.tpt_base,
1079 rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p));
1080 CTR4(KTR_IW_CXGB, "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x",
1081 rdev_p->rnic_info.pbl_base,
1082 rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
1083 rdev_p->rnic_info.rqt_top);
1084 CTR6(KTR_IW_CXGB, "udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
1085 "qpnr %d qpmask 0x%x",
1086 rdev_p->rnic_info.udbell_len,
1087 rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
1088 rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);
1090 err = cxio_hal_init_ctrl_qp(rdev_p);
1092 log(LOG_ERR, "%s error %d initializing ctrl_qp.\n",
1096 err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
1097 0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,
1100 log(LOG_ERR, "%s error %d initializing hal resources.\n",
1104 err = cxio_hal_pblpool_create(rdev_p);
1106 log(LOG_ERR, "%s error %d initializing pbl mem pool.\n",
1110 err = cxio_hal_rqtpool_create(rdev_p);
1112 log(LOG_ERR, "%s error %d initializing rqt mem pool.\n",
1118 cxio_hal_pblpool_destroy(rdev_p);
1120 cxio_hal_destroy_resource(rdev_p->rscp);
1122 cxio_hal_destroy_ctrl_qp(rdev_p);
1124 TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1129 cxio_rdev_close(struct cxio_rdev *rdev_p)
1132 cxio_hal_pblpool_destroy(rdev_p);
1133 cxio_hal_rqtpool_destroy(rdev_p);
1134 TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1135 rdev_p->t3cdev_p->ulp = NULL;
1136 cxio_hal_destroy_ctrl_qp(rdev_p);
1137 cxio_hal_destroy_resource(rdev_p->rscp);
1144 TAILQ_INIT(&rdev_list);
1146 if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))
1149 t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
1156 struct cxio_rdev *rdev, *tmp;
1158 t3_register_cpl_handler(CPL_ASYNC_NOTIF, NULL);
1159 TAILQ_FOREACH_SAFE(rdev, &rdev_list, entry, tmp)
1160 cxio_rdev_close(rdev);
1162 cxio_hal_destroy_rhdl_resource();
1167 flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
1169 struct t3_swsq *sqp;
1170 __u32 ptr = wq->sq_rptr;
1171 int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
1173 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1175 if (!sqp->signaled) {
1177 sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1178 } else if (sqp->complete) {
1181 * Insert this completed cqe into the swcq.
1183 CTR3(KTR_IW_CXGB, "%s moving cqe into swcq sq idx %ld cq idx %ld",
1184 __FUNCTION__, Q_PTR2IDX(ptr, wq->sq_size_log2),
1185 Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
1186 sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
1187 *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
1197 create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
1198 struct t3_cqe *read_cqe)
1200 read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
1201 read_cqe->len = wq->oldest_read->read_len;
1202 read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
1203 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
1204 V_CQE_OPCODE(T3_READ_REQ) |
1209 * Return a ptr to the next read wr in the SWSQ or NULL.
1212 advance_oldest_read(struct t3_wq *wq)
1215 u32 rptr = wq->oldest_read - wq->sq + 1;
1216 u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
1218 while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
1219 wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
1221 if (wq->oldest_read->opcode == T3_READ_REQ)
1225 wq->oldest_read = NULL;
1232 * check the validity of the first CQE,
1233 * supply the wq assicated with the qpid.
1235 * credit: cq credit to return to sge.
1236 * cqe_flushed: 1 iff the CQE is flushed.
1237 * cqe: copy of the polled CQE.
1241 * -1 CQE skipped, try again.
1244 cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
1245 u8 *cqe_flushed, u64 *cookie, u32 *credit)
1248 struct t3_cqe *hw_cqe, read_cqe;
1252 hw_cqe = cxio_next_cqe(cq);
1254 CTR5(KTR_IW_CXGB, "cxio_poll_cq CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x",
1255 CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
1256 CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe));
1257 CTR4(KTR_IW_CXGB, "opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
1258 CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
1259 CQE_WRID_LOW(*hw_cqe));
1262 * skip cqe's not affiliated with a QP.
1270 * Gotta tweak READ completions:
1271 * 1) the cqe doesn't contain the sq_wptr from the wr.
1272 * 2) opcode not reflected from the wr.
1273 * 3) read_len not reflected from the wr.
1274 * 4) cq_type is RQ_TYPE not SQ_TYPE.
1276 if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
1279 * Don't write to the HWCQ, so create a new read req CQE
1282 create_read_req_cqe(wq, hw_cqe, &read_cqe);
1284 advance_oldest_read(wq);
1288 * T3A: Discard TERMINATE CQEs.
1290 if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
1296 if (CQE_STATUS(*hw_cqe) || wq->error) {
1297 *cqe_flushed = wq->error;
1301 * T3A inserts errors into the CQE. We cannot return
1302 * these as work completions.
1304 /* incoming write failures */
1305 if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
1306 && RQ_TYPE(*hw_cqe)) {
1310 /* incoming read request failures */
1311 if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
1316 /* incoming SEND with no receive posted failures */
1317 if ((CQE_OPCODE(*hw_cqe) == T3_SEND) && RQ_TYPE(*hw_cqe) &&
1318 Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1328 if (RQ_TYPE(*hw_cqe)) {
1331 * HW only validates 4 bits of MSN. So we must validate that
1332 * the MSN in the SEND is the next expected MSN. If its not,
1333 * then we complete this with TPT_ERR_MSN and mark the wq in
1336 if (__predict_false((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
1338 hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
1345 * If we get here its a send completion.
1347 * Handle out of order completion. These get stuffed
1348 * in the SW SQ. Then the SW SQ is walked to move any
1349 * now in-order completions into the SW CQ. This handles
1351 * 1) reaping unsignaled WRs when the first subsequent
1352 * signaled WR is completed.
1353 * 2) out of order read completions.
1355 if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
1356 struct t3_swsq *sqp;
1358 CTR2(KTR_IW_CXGB, "%s out of order completion going in swsq at idx %ld",
1360 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
1362 Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
1373 * Reap the associated WR(s) that are freed up with this
1376 if (SQ_TYPE(*hw_cqe)) {
1377 wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
1378 CTR2(KTR_IW_CXGB, "%s completing sq idx %ld", __FUNCTION__,
1379 Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
1381 Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
1384 CTR2(KTR_IW_CXGB, "%s completing rq idx %ld", __FUNCTION__,
1385 Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1386 *cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1392 * Flush any completed cqes that are now in-order.
1394 flush_completed_wrs(wq, cq);
1397 if (SW_CQE(*hw_cqe)) {
1398 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x",
1399 __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
1402 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip hw cqe rptr 0x%x",
1403 __FUNCTION__, cq, cq->cqid, cq->rptr);
1407 * T3A: compute credits.
1409 if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
1410 || ((cq->rptr - cq->wptr) >= 128)) {
1411 *credit = cq->rptr - cq->wptr;
1412 cq->wptr = cq->rptr;