1 /**************************************************************************
3 Copyright (c) 2007, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
39 #include <sys/pciio.h>
41 #include <machine/bus.h>
42 #include <machine/resource.h>
43 #include <sys/bus_dma.h>
45 #include <sys/ioccom.h>
47 #include <sys/mutex.h>
48 #include <sys/rwlock.h>
49 #include <sys/linker.h>
50 #include <sys/firmware.h>
51 #include <sys/socket.h>
52 #include <sys/sockio.h>
54 #include <sys/sysctl.h>
55 #include <sys/syslog.h>
56 #include <sys/queue.h>
57 #include <sys/taskqueue.h>
59 #include <sys/queue.h>
61 #include <netinet/in.h>
67 #include <rdma/ib_verbs.h>
68 #include <rdma/ib_umem.h>
69 #include <rdma/ib_user_verbs.h>
70 #include <linux/idr.h>
71 #include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
74 #include <cxgb_include.h>
75 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
76 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
77 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
78 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
79 #include <ulp/iw_cxgb/iw_cxgb.h>
80 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
81 #include <ulp/iw_cxgb/iw_cxgb_user.h>
84 iwch_modify_port(struct ib_device *ibdev,
85 u8 port, int port_modify_mask,
86 struct ib_port_modify *props)
92 iwch_ah_create(struct ib_pd *pd,
93 struct ib_ah_attr *ah_attr)
95 return ERR_PTR(-ENOSYS);
99 iwch_ah_destroy(struct ib_ah *ah)
104 static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
110 iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
116 iwch_process_mad(struct ib_device *ibdev,
120 struct ib_grh *in_grh,
121 struct ib_mad *in_mad, struct ib_mad *out_mad)
127 iwch_dealloc_ucontext(struct ib_ucontext *context)
129 struct iwch_dev *rhp = to_iwch_dev(context->device);
130 struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
131 struct iwch_mm_entry *mm, *tmp;
133 CTR2(KTR_IW_CXGB, "%s context %p", __FUNCTION__, context);
134 TAILQ_FOREACH_SAFE(mm, &ucontext->mmaps, entry, tmp) {
135 TAILQ_REMOVE(&ucontext->mmaps, mm, entry);
138 cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
143 static struct ib_ucontext *
144 iwch_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
146 struct iwch_ucontext *context;
147 struct iwch_dev *rhp = to_iwch_dev(ibdev);
149 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
150 context = malloc(sizeof(*context), M_DEVBUF, M_ZERO|M_NOWAIT);
152 return ERR_PTR(-ENOMEM);
153 cxio_init_ucontext(&rhp->rdev, &context->uctx);
154 TAILQ_INIT(&context->mmaps);
155 mtx_init(&context->mmap_lock, "ucontext mmap", NULL, MTX_DEF);
156 return &context->ibucontext;
160 iwch_destroy_cq(struct ib_cq *ib_cq)
164 CTR2(KTR_IW_CXGB, "%s ib_cq %p", __FUNCTION__, ib_cq);
165 chp = to_iwch_cq(ib_cq);
167 remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
168 mtx_lock(&chp->lock);
170 msleep(chp, &chp->lock, 0, "iwch_destroy_cq", 0);
171 mtx_unlock(&chp->lock);
173 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
178 static struct ib_cq *
179 iwch_create_cq(struct ib_device *ibdev, int entries, int vector,
180 struct ib_ucontext *ib_context,
181 struct ib_udata *udata)
183 struct iwch_dev *rhp;
185 struct iwch_create_cq_resp uresp;
186 struct iwch_create_cq_req ureq;
187 struct iwch_ucontext *ucontext = NULL;
191 CTR3(KTR_IW_CXGB, "%s ib_dev %p entries %d", __FUNCTION__, ibdev, entries);
192 rhp = to_iwch_dev(ibdev);
193 chp = malloc(sizeof(*chp), M_DEVBUF, M_NOWAIT|M_ZERO);
195 return ERR_PTR(-ENOMEM);
198 ucontext = to_iwch_ucontext(ib_context);
199 if (!t3a_device(rhp)) {
200 if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
202 return ERR_PTR(-EFAULT);
204 chp->user_rptr_addr = (u32 /*__user */*)(unsigned long)ureq.user_rptr_addr;
208 if (t3a_device(rhp)) {
211 * T3A: Add some fluff to handle extra CQEs inserted
212 * for various errors.
213 * Additional CQE possibilities:
215 * incoming RDMA WRITE Failures
216 * incoming RDMA READ REQUEST FAILUREs
217 * NOTE: We cannot ensure the CQ won't overflow.
221 entries = roundup_pow_of_two(entries);
222 chp->cq.size_log2 = ilog2(entries);
224 if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
226 return ERR_PTR(-ENOMEM);
229 chp->ibcq.cqe = 1 << chp->cq.size_log2;
230 mtx_init(&chp->lock, "cxgb cq", NULL, MTX_DEF|MTX_DUPOK);
232 if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
233 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
235 return ERR_PTR(-ENOMEM);
239 struct iwch_mm_entry *mm;
241 mm = kmalloc(sizeof *mm, M_NOWAIT);
243 iwch_destroy_cq(&chp->ibcq);
244 return ERR_PTR(-ENOMEM);
246 uresp.cqid = chp->cq.cqid;
247 uresp.size_log2 = chp->cq.size_log2;
248 mtx_lock(&ucontext->mmap_lock);
249 uresp.key = ucontext->key;
250 ucontext->key += PAGE_SIZE;
251 mtx_unlock(&ucontext->mmap_lock);
253 mm->addr = vtophys(chp->cq.queue);
254 if (udata->outlen < sizeof uresp) {
256 CTR1(KTR_IW_CXGB, "%s Warning - "
257 "downlevel libcxgb3 (non-fatal).\n",
259 mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
260 sizeof(struct t3_cqe));
261 resplen = sizeof(struct iwch_create_cq_resp_v0);
263 mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
264 sizeof(struct t3_cqe));
265 uresp.memsize = mm->len;
266 resplen = sizeof uresp;
268 if (ib_copy_to_udata(udata, &uresp, resplen)) {
270 iwch_destroy_cq(&chp->ibcq);
271 return ERR_PTR(-EFAULT);
273 insert_mmap(ucontext, mm);
275 CTR4(KTR_IW_CXGB, "created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx",
276 chp->cq.cqid, chp, (1 << chp->cq.size_log2),
277 (unsigned long long) chp->cq.dma_addr);
282 iwch_resize_cq(struct ib_cq *cq __unused, int cqe __unused,
283 struct ib_udata *udata __unused)
290 iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
292 struct iwch_dev *rhp;
294 enum t3_cq_opcode cq_op;
298 chp = to_iwch_cq(ibcq);
300 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
304 if (chp->user_rptr_addr) {
305 if (copyin(&rptr, chp->user_rptr_addr, 4))
307 mtx_lock(&chp->lock);
310 mtx_lock(&chp->lock);
311 CTR2(KTR_IW_CXGB, "%s rptr 0x%x", __FUNCTION__, chp->cq.rptr);
312 err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
313 mtx_unlock(&chp->lock);
315 log(LOG_ERR, "Error %d rearming CQID 0x%x\n", err,
317 if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
323 iwch_mmap(struct ib_ucontext *context __unused, struct vm_area_struct *vma __unused)
329 static int iwch_deallocate_pd(struct ib_pd *pd)
331 struct iwch_dev *rhp;
334 php = to_iwch_pd(pd);
336 CTR3(KTR_IW_CXGB, "%s ibpd %p pdid 0x%x", __FUNCTION__, pd, php->pdid);
337 cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
342 static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
343 struct ib_ucontext *context,
344 struct ib_udata *udata)
348 struct iwch_dev *rhp;
350 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
351 rhp = (struct iwch_dev *) ibdev;
352 pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
354 return ERR_PTR(-EINVAL);
355 php = malloc(sizeof(*php), M_DEVBUF, M_ZERO|M_NOWAIT);
357 cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
358 return ERR_PTR(-ENOMEM);
363 if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
364 iwch_deallocate_pd(&php->ibpd);
365 return ERR_PTR(-EFAULT);
368 CTR3(KTR_IW_CXGB, "%s pdid 0x%0x ptr 0x%p", __FUNCTION__, pdid, php);
372 static int iwch_dereg_mr(struct ib_mr *ib_mr)
374 struct iwch_dev *rhp;
378 CTR2(KTR_IW_CXGB, "%s ib_mr %p", __FUNCTION__, ib_mr);
379 /* There can be no memory windows */
380 if (atomic_load_acq_int(&ib_mr->usecnt.counter))
383 mhp = to_iwch_mr(ib_mr);
385 mmid = mhp->attr.stag >> 8;
386 cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
389 remove_handle(rhp, &rhp->mmidr, mmid);
391 cxfree((void *) (unsigned long) mhp->kva);
393 ib_umem_release(mhp->umem);
394 CTR3(KTR_IW_CXGB, "%s mmid 0x%x ptr %p", __FUNCTION__, mmid, mhp);
399 static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
400 struct ib_phys_buf *buffer_list,
409 struct iwch_dev *rhp;
414 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
415 php = to_iwch_pd(pd);
418 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
420 return ERR_PTR(-ENOMEM);
424 /* First check that we have enough alignment */
425 if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
430 if (num_phys_buf > 1 &&
431 ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
436 ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
437 &total_size, &npages, &shift, &page_list);
441 ret = iwch_alloc_pbl(mhp, npages);
447 ret = iwch_write_pbl(mhp, page_list, npages, 0);
452 mhp->attr.pdid = php->pdid;
455 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
456 mhp->attr.va_fbo = *iova_start;
457 mhp->attr.page_size = shift - 12;
459 mhp->attr.len = (u32) total_size;
460 mhp->attr.pbl_size = npages;
461 ret = iwch_register_mem(rhp, php, mhp, shift);
476 static int iwch_reregister_phys_mem(struct ib_mr *mr,
479 struct ib_phys_buf *buffer_list,
481 int acc, u64 * iova_start)
484 struct iwch_mr mh, *mhp;
486 struct iwch_dev *rhp;
487 __be64 *page_list = NULL;
493 CTR3(KTR_IW_CXGB, "%s ib_mr %p ib_pd %p", __FUNCTION__, mr, pd);
495 /* There can be no memory windows */
496 if (atomic_load_acq_int(&mr->usecnt.counter))
499 mhp = to_iwch_mr(mr);
501 php = to_iwch_pd(mr->pd);
503 /* make sure we are on the same adapter */
507 memcpy(&mh, mhp, sizeof *mhp);
509 if (mr_rereg_mask & IB_MR_REREG_PD)
510 php = to_iwch_pd(pd);
511 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
512 mh.attr.perms = iwch_ib_to_tpt_access(acc);
513 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
514 ret = build_phys_page_list(buffer_list, num_phys_buf,
516 &total_size, &npages,
522 ret = iwch_reregister_mem(rhp, php, &mh, shift, npages);
527 if (mr_rereg_mask & IB_MR_REREG_PD)
528 mhp->attr.pdid = php->pdid;
529 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
530 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
531 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
533 mhp->attr.va_fbo = *iova_start;
534 mhp->attr.page_size = shift - 12;
535 mhp->attr.len = (u32) total_size;
536 mhp->attr.pbl_size = npages;
543 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
544 u64 virt, int acc, struct ib_udata *udata)
549 struct ib_umem_chunk *chunk;
550 struct iwch_dev *rhp;
553 struct iwch_reg_user_mr_resp uresp;
558 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
560 php = to_iwch_pd(pd);
562 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_NOWAIT|M_ZERO);
564 return ERR_PTR(-ENOMEM);
568 mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
569 if (IS_ERR(mhp->umem)) {
570 err = PTR_ERR(mhp->umem);
572 return ERR_PTR(-err);
575 shift = ffs(mhp->umem->page_size) - 1;
578 list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
581 err = iwch_alloc_pbl(mhp, n);
585 pages = (__be64 *) kmalloc(n * sizeof(u64), M_NOWAIT);
594 TAILQ_FOREACH(chunk, &mhp->umem->chunk_list, entry)
595 for (j = 0; j < chunk->nmap; ++j) {
596 len = sg_dma_len(&chunk->page_list[j]) >> shift;
597 for (k = 0; k < len; ++k) {
598 pages[i++] = htobe64(sg_dma_address(
599 &chunk->page_list[j]) +
600 mhp->umem->page_size * k);
601 if (i == PAGE_SIZE / sizeof *pages) {
602 err = iwch_write_pbl(mhp, pages, i, n);
613 err = iwch_write_pbl(mhp, pages, i, n);
621 mhp->attr.pdid = php->pdid;
623 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
624 mhp->attr.va_fbo = virt;
625 mhp->attr.page_size = shift - 12;
626 mhp->attr.len = (u32) length;
628 err = iwch_register_mem(rhp, php, mhp, shift);
632 if (udata && !t3a_device(rhp)) {
633 uresp.pbl_addr = (mhp->attr.pbl_addr -
634 rhp->rdev.rnic_info.pbl_base) >> 3;
635 CTR2(KTR_IW_CXGB, "%s user resp pbl_addr 0x%x", __FUNCTION__,
638 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
639 iwch_dereg_mr(&mhp->ibmr);
651 ib_umem_release(mhp->umem);
653 return ERR_PTR(-err);
656 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
658 struct ib_phys_buf bl;
662 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
665 * T3 only supports 32 bits of size.
667 bl.size = 0xffffffff;
670 ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva);
674 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd)
676 struct iwch_dev *rhp;
683 php = to_iwch_pd(pd);
685 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
687 return ERR_PTR(-ENOMEM);
688 ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
691 return ERR_PTR(-ret);
694 mhp->attr.pdid = php->pdid;
695 mhp->attr.type = TPT_MW;
696 mhp->attr.stag = stag;
698 mhp->ibmw.rkey = stag;
699 if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
700 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
702 return ERR_PTR(-ENOMEM);
704 CTR4(KTR_IW_CXGB, "%s mmid 0x%x mhp %p stag 0x%x", __FUNCTION__, mmid, mhp, stag);
708 static int iwch_dealloc_mw(struct ib_mw *mw)
710 struct iwch_dev *rhp;
714 mhp = to_iwch_mw(mw);
716 mmid = (mw->rkey) >> 8;
717 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
718 remove_handle(rhp, &rhp->mmidr, mmid);
720 CTR4(KTR_IW_CXGB, "%s ib_mw %p mmid 0x%x ptr %p", __FUNCTION__, mw, mmid, mhp);
724 static int iwch_destroy_qp(struct ib_qp *ib_qp)
726 struct iwch_dev *rhp;
728 struct iwch_qp_attributes attrs;
729 struct iwch_ucontext *ucontext;
731 qhp = to_iwch_qp(ib_qp);
734 attrs.next_state = IWCH_QP_STATE_ERROR;
735 iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
736 mtx_lock(&qhp->lock);
738 msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp1", 0);
739 mtx_unlock(&qhp->lock);
741 remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
743 mtx_lock(&qhp->lock);
745 msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp2", 0);
746 mtx_unlock(&qhp->lock);
748 ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
750 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
751 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
753 CTR4(KTR_IW_CXGB, "%s ib_qp %p qpid 0x%0x qhp %p", __FUNCTION__,
754 ib_qp, qhp->wq.qpid, qhp);
759 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
760 struct ib_qp_init_attr *attrs,
761 struct ib_udata *udata)
763 struct iwch_dev *rhp;
766 struct iwch_cq *schp;
767 struct iwch_cq *rchp;
768 struct iwch_create_qp_resp uresp;
769 int wqsize, sqsize, rqsize;
770 struct iwch_ucontext *ucontext;
772 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
773 if (attrs->qp_type != IB_QPT_RC)
774 return ERR_PTR(-EINVAL);
775 php = to_iwch_pd(pd);
777 schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
778 rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
780 return ERR_PTR(-EINVAL);
782 /* The RQT size must be # of entries + 1 rounded up to a power of two */
783 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
784 if (rqsize == attrs->cap.max_recv_wr)
785 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
787 /* T3 doesn't support RQT depth < 16 */
791 if (rqsize > T3_MAX_RQ_SIZE)
792 return ERR_PTR(-EINVAL);
794 if (attrs->cap.max_inline_data > T3_MAX_INLINE)
795 return ERR_PTR(-EINVAL);
798 * NOTE: The SQ and total WQ sizes don't need to be
799 * a power of two. However, all the code assumes
800 * they are. EG: Q_FREECNT() and friends.
802 sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
803 wqsize = roundup_pow_of_two(rqsize + sqsize);
804 CTR4(KTR_IW_CXGB, "%s wqsize %d sqsize %d rqsize %d", __FUNCTION__,
805 wqsize, sqsize, rqsize);
806 qhp = malloc(sizeof(*qhp), M_DEVBUF, M_ZERO|M_NOWAIT);
808 return ERR_PTR(-ENOMEM);
809 qhp->wq.size_log2 = ilog2(wqsize);
810 qhp->wq.rq_size_log2 = ilog2(rqsize);
811 qhp->wq.sq_size_log2 = ilog2(sqsize);
812 ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
813 if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
814 ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
816 return ERR_PTR(-ENOMEM);
819 attrs->cap.max_recv_wr = rqsize - 1;
820 attrs->cap.max_send_wr = sqsize;
821 attrs->cap.max_inline_data = T3_MAX_INLINE;
824 qhp->attr.pd = php->pdid;
825 qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
826 qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
827 qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
828 qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
829 qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
830 qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
831 qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
832 qhp->attr.state = IWCH_QP_STATE_IDLE;
833 qhp->attr.next_state = IWCH_QP_STATE_IDLE;
836 * XXX - These don't get passed in from the openib user
837 * at create time. The CM sets them via a QP modify.
838 * Need to fix... I think the CM should
840 qhp->attr.enable_rdma_read = 1;
841 qhp->attr.enable_rdma_write = 1;
842 qhp->attr.enable_bind = 1;
843 qhp->attr.max_ord = 1;
844 qhp->attr.max_ird = 1;
846 mtx_init(&qhp->lock, "cxgb qp", NULL, MTX_DEF|MTX_DUPOK);
849 if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
850 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
851 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
853 return ERR_PTR(-ENOMEM);
858 struct iwch_mm_entry *mm1, *mm2;
860 mm1 = kmalloc(sizeof *mm1, M_NOWAIT);
862 iwch_destroy_qp(&qhp->ibqp);
863 return ERR_PTR(-ENOMEM);
866 mm2 = kmalloc(sizeof *mm2, M_NOWAIT);
869 iwch_destroy_qp(&qhp->ibqp);
870 return ERR_PTR(-ENOMEM);
873 uresp.qpid = qhp->wq.qpid;
874 uresp.size_log2 = qhp->wq.size_log2;
875 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
876 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
877 mtx_lock(&ucontext->mmap_lock);
878 uresp.key = ucontext->key;
879 ucontext->key += PAGE_SIZE;
880 uresp.db_key = ucontext->key;
881 ucontext->key += PAGE_SIZE;
882 mtx_unlock(&ucontext->mmap_lock);
883 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
886 iwch_destroy_qp(&qhp->ibqp);
887 return ERR_PTR(-EFAULT);
889 mm1->key = uresp.key;
890 mm1->addr = vtophys(qhp->wq.queue);
891 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
892 insert_mmap(ucontext, mm1);
893 mm2->key = uresp.db_key;
894 mm2->addr = qhp->wq.udb & PAGE_MASK;
895 mm2->len = PAGE_SIZE;
896 insert_mmap(ucontext, mm2);
898 qhp->ibqp.qp_num = qhp->wq.qpid;
899 callout_init(&(qhp->timer), TRUE);
900 CTR6(KTR_IW_CXGB, "sq_num_entries %d, rq_num_entries %d "
901 "qpid 0x%0x qhp %p dma_addr 0x%llx size %d",
902 qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
903 qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
904 1 << qhp->wq.size_log2);
908 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
909 int attr_mask, struct ib_udata *udata)
911 struct iwch_dev *rhp;
913 enum iwch_qp_attr_mask mask = 0;
914 struct iwch_qp_attributes attrs;
916 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, ibqp);
918 /* iwarp does not support the RTR state */
919 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
920 attr_mask &= ~IB_QP_STATE;
922 /* Make sure we still have something left to do */
926 memset(&attrs, 0, sizeof attrs);
927 qhp = to_iwch_qp(ibqp);
930 attrs.next_state = iwch_convert_state(attr->qp_state);
931 attrs.enable_rdma_read = (attr->qp_access_flags &
932 IB_ACCESS_REMOTE_READ) ? 1 : 0;
933 attrs.enable_rdma_write = (attr->qp_access_flags &
934 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
935 attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
938 mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
939 mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
940 (IWCH_QP_ATTR_ENABLE_RDMA_READ |
941 IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
942 IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
944 return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
947 void iwch_qp_add_ref(struct ib_qp *qp)
949 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
950 mtx_lock(&to_iwch_qp(qp)->lock);
951 to_iwch_qp(qp)->refcnt++;
952 mtx_unlock(&to_iwch_qp(qp)->lock);
955 void iwch_qp_rem_ref(struct ib_qp *qp)
957 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
958 mtx_lock(&to_iwch_qp(qp)->lock);
959 if (--to_iwch_qp(qp)->refcnt == 0)
960 wakeup(to_iwch_qp(qp));
961 mtx_unlock(&to_iwch_qp(qp)->lock);
964 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
966 CTR3(KTR_IW_CXGB, "%s ib_dev %p qpn 0x%x", __FUNCTION__, dev, qpn);
967 return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
971 static int iwch_query_pkey(struct ib_device *ibdev,
972 u8 port, u16 index, u16 * pkey)
974 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
979 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
980 int index, union ib_gid *gid)
982 struct iwch_dev *dev;
983 struct port_info *pi;
986 CTR5(KTR_IW_CXGB, "%s ibdev %p, port %d, index %d, gid %p",
987 __FUNCTION__, ibdev, port, index, gid);
988 dev = to_iwch_dev(ibdev);
990 PANIC_IF(port == 0 || port > 2);
991 pi = &sc->port[port - 1];
992 memset(&(gid->raw[0]), 0, sizeof(gid->raw));
993 memcpy(&(gid->raw[0]), pi->hw_addr, 6);
997 static int iwch_query_device(struct ib_device *ibdev,
998 struct ib_device_attr *props)
1000 struct iwch_dev *dev;
1003 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1005 dev = to_iwch_dev(ibdev);
1006 sc = dev->rdev.adap;
1007 memset(props, 0, sizeof *props);
1008 memcpy(&props->sys_image_guid, sc->port[0].hw_addr, 6);
1009 props->device_cap_flags = dev->device_cap_flags;
1010 props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1011 props->vendor_id = pci_get_vendor(sc->dev);
1012 props->vendor_part_id = pci_get_device(sc->dev);
1013 props->max_mr_size = dev->attr.max_mr_size;
1014 props->max_qp = dev->attr.max_qps;
1015 props->max_qp_wr = dev->attr.max_wrs;
1016 props->max_sge = dev->attr.max_sge_per_wr;
1017 props->max_sge_rd = 1;
1018 props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1019 props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1020 props->max_cq = dev->attr.max_cqs;
1021 props->max_cqe = dev->attr.max_cqes_per_cq;
1022 props->max_mr = dev->attr.max_mem_regs;
1023 props->max_pd = dev->attr.max_pds;
1024 props->local_ca_ack_delay = 0;
1029 static int iwch_query_port(struct ib_device *ibdev,
1030 u8 port, struct ib_port_attr *props)
1032 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1033 memset(props, 0, sizeof(struct ib_port_attr));
1034 props->max_mtu = IB_MTU_4096;
1035 props->active_mtu = IB_MTU_2048;
1036 props->state = IB_PORT_ACTIVE;
1037 props->port_cap_flags =
1039 IB_PORT_SNMP_TUNNEL_SUP |
1040 IB_PORT_REINIT_SUP |
1041 IB_PORT_DEVICE_MGMT_SUP |
1042 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1043 props->gid_tbl_len = 1;
1044 props->pkey_tbl_len = 1;
1045 props->active_width = 2;
1046 props->active_speed = 2;
1047 props->max_msg_sz = -1;
1052 int iwch_register_device(struct iwch_dev *dev)
1055 struct adapter *sc = dev->rdev.adap;
1057 CTR2(KTR_IW_CXGB, "%s iwch_dev %p", __FUNCTION__, dev);
1058 strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1059 memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1060 memcpy(&dev->ibdev.node_guid, sc->port[0].hw_addr, 6);
1061 dev->device_cap_flags =
1062 (IB_DEVICE_LOCAL_DMA_LKEY |
1063 IB_DEVICE_MEM_WINDOW);
1065 dev->ibdev.uverbs_cmd_mask =
1066 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1067 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1068 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1069 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1070 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1071 (1ull << IB_USER_VERBS_CMD_REG_MR) |
1072 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1073 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1074 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1075 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1076 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1077 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1078 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1079 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1080 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1081 (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1082 (1ull << IB_USER_VERBS_CMD_POST_RECV);
1083 dev->ibdev.node_type = RDMA_NODE_RNIC;
1084 memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1085 dev->ibdev.phys_port_cnt = sc->params.nports;
1086 dev->ibdev.num_comp_vectors = 1;
1087 dev->ibdev.dma_device = dev->rdev.adap->dev;
1088 dev->ibdev.query_device = iwch_query_device;
1089 dev->ibdev.query_port = iwch_query_port;
1090 dev->ibdev.modify_port = iwch_modify_port;
1091 dev->ibdev.query_pkey = iwch_query_pkey;
1092 dev->ibdev.query_gid = iwch_query_gid;
1093 dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1094 dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1095 dev->ibdev.mmap = iwch_mmap;
1096 dev->ibdev.alloc_pd = iwch_allocate_pd;
1097 dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1098 dev->ibdev.create_ah = iwch_ah_create;
1099 dev->ibdev.destroy_ah = iwch_ah_destroy;
1100 dev->ibdev.create_qp = iwch_create_qp;
1101 dev->ibdev.modify_qp = iwch_ib_modify_qp;
1102 dev->ibdev.destroy_qp = iwch_destroy_qp;
1103 dev->ibdev.create_cq = iwch_create_cq;
1104 dev->ibdev.destroy_cq = iwch_destroy_cq;
1105 dev->ibdev.resize_cq = iwch_resize_cq;
1106 dev->ibdev.poll_cq = iwch_poll_cq;
1107 dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1108 dev->ibdev.reg_phys_mr = iwch_register_phys_mem;
1109 dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem;
1110 dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1111 dev->ibdev.dereg_mr = iwch_dereg_mr;
1112 dev->ibdev.alloc_mw = iwch_alloc_mw;
1113 dev->ibdev.bind_mw = iwch_bind_mw;
1114 dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1116 dev->ibdev.attach_mcast = iwch_multicast_attach;
1117 dev->ibdev.detach_mcast = iwch_multicast_detach;
1118 dev->ibdev.process_mad = iwch_process_mad;
1120 dev->ibdev.req_notify_cq = iwch_arm_cq;
1121 dev->ibdev.post_send = iwch_post_send;
1122 dev->ibdev.post_recv = iwch_post_receive;
1123 dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1126 kmalloc(sizeof(struct iw_cm_verbs), M_NOWAIT);
1127 if (!dev->ibdev.iwcm)
1130 dev->ibdev.iwcm->connect = iwch_connect;
1131 dev->ibdev.iwcm->accept = iwch_accept_cr;
1132 dev->ibdev.iwcm->reject = iwch_reject_cr;
1133 dev->ibdev.iwcm->create_listen = iwch_create_listen;
1134 dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1135 dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1136 dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1137 dev->ibdev.iwcm->get_qp = iwch_get_qp;
1139 ret = ib_register_device(&dev->ibdev);
1146 cxfree(dev->ibdev.iwcm);
1150 void iwch_unregister_device(struct iwch_dev *dev)
1153 ib_unregister_device(&dev->ibdev);
1154 cxfree(dev->ibdev.iwcm);