1 /**************************************************************************
3 Copyright (c) 2007, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
39 #include <sys/pciio.h>
41 #include <machine/bus.h>
42 #include <machine/resource.h>
43 #include <sys/bus_dma.h>
45 #include <sys/ioccom.h>
47 #include <sys/mutex.h>
48 #include <sys/rwlock.h>
49 #include <sys/linker.h>
50 #include <sys/firmware.h>
51 #include <sys/socket.h>
52 #include <sys/sockio.h>
54 #include <sys/sysctl.h>
55 #include <sys/syslog.h>
56 #include <sys/queue.h>
57 #include <sys/taskqueue.h>
59 #include <sys/queue.h>
61 #include <netinet/in.h>
67 #include <rdma/ib_verbs.h>
68 #include <rdma/ib_umem.h>
69 #include <rdma/ib_user_verbs.h>
70 #include <linux/idr.h>
71 #include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
74 #include <cxgb_include.h>
75 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
76 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
77 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
78 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
79 #include <ulp/iw_cxgb/iw_cxgb.h>
80 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
81 #include <ulp/iw_cxgb/iw_cxgb_user.h>
84 iwch_modify_port(struct ib_device *ibdev,
85 u8 port, int port_modify_mask,
86 struct ib_port_modify *props)
92 iwch_ah_create(struct ib_pd *pd,
93 struct ib_ah_attr *ah_attr)
95 return ERR_PTR(-ENOSYS);
99 iwch_ah_destroy(struct ib_ah *ah)
104 static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
110 iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
116 iwch_process_mad(struct ib_device *ibdev,
120 struct ib_grh *in_grh,
121 struct ib_mad *in_mad, struct ib_mad *out_mad)
127 iwch_dealloc_ucontext(struct ib_ucontext *context)
129 struct iwch_dev *rhp = to_iwch_dev(context->device);
130 struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
131 struct iwch_mm_entry *mm, *tmp;
133 CTR2(KTR_IW_CXGB, "%s context %p", __FUNCTION__, context);
134 TAILQ_FOREACH_SAFE(mm, &ucontext->mmaps, entry, tmp) {
135 TAILQ_REMOVE(&ucontext->mmaps, mm, entry);
138 cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
143 static struct ib_ucontext *
144 iwch_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
146 struct iwch_ucontext *context;
147 struct iwch_dev *rhp = to_iwch_dev(ibdev);
149 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
150 context = malloc(sizeof(*context), M_DEVBUF, M_ZERO|M_NOWAIT);
152 return ERR_PTR(-ENOMEM);
153 cxio_init_ucontext(&rhp->rdev, &context->uctx);
154 TAILQ_INIT(&context->mmaps);
155 mtx_init(&context->mmap_lock, "ucontext mmap", NULL, MTX_DEF);
156 return &context->ibucontext;
160 iwch_destroy_cq(struct ib_cq *ib_cq)
164 CTR2(KTR_IW_CXGB, "%s ib_cq %p", __FUNCTION__, ib_cq);
165 chp = to_iwch_cq(ib_cq);
167 remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
168 mtx_lock(&chp->lock);
170 msleep(chp, &chp->lock, 0, "iwch_destroy_cq", 0);
171 mtx_unlock(&chp->lock);
173 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
178 static struct ib_cq *
179 iwch_create_cq(struct ib_device *ibdev, int entries, int vector,
180 struct ib_ucontext *ib_context,
181 struct ib_udata *udata)
183 struct iwch_dev *rhp;
185 struct iwch_create_cq_resp uresp;
186 struct iwch_create_cq_req ureq;
187 struct iwch_ucontext *ucontext = NULL;
191 CTR3(KTR_IW_CXGB, "%s ib_dev %p entries %d", __FUNCTION__, ibdev, entries);
192 rhp = to_iwch_dev(ibdev);
193 chp = malloc(sizeof(*chp), M_DEVBUF, M_NOWAIT|M_ZERO);
195 return ERR_PTR(-ENOMEM);
198 ucontext = to_iwch_ucontext(ib_context);
199 if (!t3a_device(rhp)) {
200 if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
202 return ERR_PTR(-EFAULT);
204 chp->user_rptr_addr = (u32 /*__user */*)(unsigned long)ureq.user_rptr_addr;
208 if (t3a_device(rhp)) {
211 * T3A: Add some fluff to handle extra CQEs inserted
212 * for various errors.
213 * Additional CQE possibilities:
215 * incoming RDMA WRITE Failures
216 * incoming RDMA READ REQUEST FAILUREs
217 * NOTE: We cannot ensure the CQ won't overflow.
221 entries = roundup_pow_of_two(entries);
222 chp->cq.size_log2 = ilog2(entries);
224 if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
226 return ERR_PTR(-ENOMEM);
229 chp->ibcq.cqe = 1 << chp->cq.size_log2;
230 mtx_init(&chp->lock, "cxgb cq", NULL, MTX_DEF|MTX_DUPOK);
232 if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
233 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
235 return ERR_PTR(-ENOMEM);
239 struct iwch_mm_entry *mm;
241 mm = kmalloc(sizeof *mm, M_NOWAIT);
243 iwch_destroy_cq(&chp->ibcq);
244 return ERR_PTR(-ENOMEM);
246 uresp.cqid = chp->cq.cqid;
247 uresp.size_log2 = chp->cq.size_log2;
248 mtx_lock(&ucontext->mmap_lock);
249 uresp.key = ucontext->key;
250 ucontext->key += PAGE_SIZE;
251 mtx_unlock(&ucontext->mmap_lock);
253 mm->addr = vtophys(chp->cq.queue);
254 if (udata->outlen < sizeof uresp) {
256 CTR1(KTR_IW_CXGB, "%s Warning - "
257 "downlevel libcxgb3 (non-fatal).\n",
259 mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
260 sizeof(struct t3_cqe));
261 resplen = sizeof(struct iwch_create_cq_resp_v0);
263 mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
264 sizeof(struct t3_cqe));
265 uresp.memsize = mm->len;
266 resplen = sizeof uresp;
268 if (ib_copy_to_udata(udata, &uresp, resplen)) {
270 iwch_destroy_cq(&chp->ibcq);
271 return ERR_PTR(-EFAULT);
273 insert_mmap(ucontext, mm);
275 CTR4(KTR_IW_CXGB, "created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx",
276 chp->cq.cqid, chp, (1 << chp->cq.size_log2),
277 (unsigned long long) chp->cq.dma_addr);
282 iwch_resize_cq(struct ib_cq *cq __unused, int cqe __unused,
283 struct ib_udata *udata __unused)
290 iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
292 struct iwch_dev *rhp;
294 enum t3_cq_opcode cq_op;
298 chp = to_iwch_cq(ibcq);
300 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
304 if (chp->user_rptr_addr) {
305 if (copyin(&rptr, chp->user_rptr_addr, 4))
307 mtx_lock(&chp->lock);
310 mtx_lock(&chp->lock);
311 CTR2(KTR_IW_CXGB, "%s rptr 0x%x", __FUNCTION__, chp->cq.rptr);
312 err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
313 mtx_unlock(&chp->lock);
315 log(LOG_ERR, "Error %d rearming CQID 0x%x\n", err,
317 if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
323 iwch_mmap(struct ib_ucontext *context __unused, struct vm_area_struct *vma __unused)
329 static int iwch_deallocate_pd(struct ib_pd *pd)
331 struct iwch_dev *rhp;
334 php = to_iwch_pd(pd);
336 CTR3(KTR_IW_CXGB, "%s ibpd %p pdid 0x%x", __FUNCTION__, pd, php->pdid);
337 cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
342 static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
343 struct ib_ucontext *context,
344 struct ib_udata *udata)
348 struct iwch_dev *rhp;
350 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
351 rhp = (struct iwch_dev *) ibdev;
352 pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
354 return ERR_PTR(-EINVAL);
355 php = malloc(sizeof(*php), M_DEVBUF, M_ZERO|M_NOWAIT);
357 cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
358 return ERR_PTR(-ENOMEM);
363 if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
364 iwch_deallocate_pd(&php->ibpd);
365 return ERR_PTR(-EFAULT);
368 CTR3(KTR_IW_CXGB, "%s pdid 0x%0x ptr 0x%p", __FUNCTION__, pdid, php);
372 static int iwch_dereg_mr(struct ib_mr *ib_mr)
374 struct iwch_dev *rhp;
378 CTR2(KTR_IW_CXGB, "%s ib_mr %p", __FUNCTION__, ib_mr);
379 /* There can be no memory windows */
380 if (atomic_load_acq_int(&ib_mr->usecnt.counter))
383 mhp = to_iwch_mr(ib_mr);
385 mmid = mhp->attr.stag >> 8;
386 cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
389 remove_handle(rhp, &rhp->mmidr, mmid);
391 cxfree((void *) (unsigned long) mhp->kva);
393 ib_umem_release(mhp->umem);
394 CTR3(KTR_IW_CXGB, "%s mmid 0x%x ptr %p", __FUNCTION__, mmid, mhp);
399 static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
400 struct ib_phys_buf *buffer_list,
409 struct iwch_dev *rhp;
414 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
415 php = to_iwch_pd(pd);
418 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
420 return ERR_PTR(-ENOMEM);
424 /* First check that we have enough alignment */
425 if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
430 if (num_phys_buf > 1 &&
431 ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
436 ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
437 &total_size, &npages, &shift, &page_list);
441 ret = iwch_alloc_pbl(mhp, npages);
447 ret = iwch_write_pbl(mhp, page_list, npages, 0);
452 mhp->attr.pdid = php->pdid;
455 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
456 mhp->attr.va_fbo = *iova_start;
457 mhp->attr.page_size = shift - 12;
459 mhp->attr.len = (u32) total_size;
460 mhp->attr.pbl_size = npages;
461 ret = iwch_register_mem(rhp, php, mhp, shift);
476 static int iwch_reregister_phys_mem(struct ib_mr *mr,
479 struct ib_phys_buf *buffer_list,
481 int acc, u64 * iova_start)
484 struct iwch_mr mh, *mhp;
486 struct iwch_dev *rhp;
487 __be64 *page_list = NULL;
493 CTR3(KTR_IW_CXGB, "%s ib_mr %p ib_pd %p", __FUNCTION__, mr, pd);
495 /* There can be no memory windows */
496 if (atomic_load_acq_int(&mr->usecnt.counter))
499 mhp = to_iwch_mr(mr);
501 php = to_iwch_pd(mr->pd);
503 /* make sure we are on the same adapter */
507 memcpy(&mh, mhp, sizeof *mhp);
509 if (mr_rereg_mask & IB_MR_REREG_PD)
510 php = to_iwch_pd(pd);
511 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
512 mh.attr.perms = iwch_ib_to_tpt_access(acc);
513 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
514 ret = build_phys_page_list(buffer_list, num_phys_buf,
516 &total_size, &npages,
522 ret = iwch_reregister_mem(rhp, php, &mh, shift, npages);
527 if (mr_rereg_mask & IB_MR_REREG_PD)
528 mhp->attr.pdid = php->pdid;
529 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
530 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
531 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
533 mhp->attr.va_fbo = *iova_start;
534 mhp->attr.page_size = shift - 12;
535 mhp->attr.len = (u32) total_size;
536 mhp->attr.pbl_size = npages;
543 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
544 u64 virt, int acc, struct ib_udata *udata,
550 struct ib_umem_chunk *chunk;
551 struct iwch_dev *rhp;
554 struct iwch_reg_user_mr_resp uresp;
559 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
561 php = to_iwch_pd(pd);
563 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_NOWAIT|M_ZERO);
565 return ERR_PTR(-ENOMEM);
569 mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
570 if (IS_ERR(mhp->umem)) {
571 err = PTR_ERR(mhp->umem);
573 return ERR_PTR(-err);
576 shift = ffs(mhp->umem->page_size) - 1;
579 list_for_each_entry(chunk, &mhp->umem->chunk_list, list)
582 err = iwch_alloc_pbl(mhp, n);
586 pages = (__be64 *) kmalloc(n * sizeof(u64), M_NOWAIT);
595 TAILQ_FOREACH(chunk, &mhp->umem->chunk_list, entry)
596 for (j = 0; j < chunk->nmap; ++j) {
597 len = sg_dma_len(&chunk->page_list[j]) >> shift;
598 for (k = 0; k < len; ++k) {
599 pages[i++] = htobe64(sg_dma_address(
600 &chunk->page_list[j]) +
601 mhp->umem->page_size * k);
602 if (i == PAGE_SIZE / sizeof *pages) {
603 err = iwch_write_pbl(mhp, pages, i, n);
614 err = iwch_write_pbl(mhp, pages, i, n);
622 mhp->attr.pdid = php->pdid;
624 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
625 mhp->attr.va_fbo = virt;
626 mhp->attr.page_size = shift - 12;
627 mhp->attr.len = (u32) length;
629 err = iwch_register_mem(rhp, php, mhp, shift);
633 if (udata && !t3a_device(rhp)) {
634 uresp.pbl_addr = (mhp->attr.pbl_addr -
635 rhp->rdev.rnic_info.pbl_base) >> 3;
636 CTR2(KTR_IW_CXGB, "%s user resp pbl_addr 0x%x", __FUNCTION__,
639 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
640 iwch_dereg_mr(&mhp->ibmr);
652 ib_umem_release(mhp->umem);
654 return ERR_PTR(-err);
657 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
659 struct ib_phys_buf bl;
663 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
666 * T3 only supports 32 bits of size.
668 bl.size = 0xffffffff;
671 ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva);
675 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd)
677 struct iwch_dev *rhp;
684 php = to_iwch_pd(pd);
686 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
688 return ERR_PTR(-ENOMEM);
689 ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
692 return ERR_PTR(-ret);
695 mhp->attr.pdid = php->pdid;
696 mhp->attr.type = TPT_MW;
697 mhp->attr.stag = stag;
699 mhp->ibmw.rkey = stag;
700 if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
701 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
703 return ERR_PTR(-ENOMEM);
705 CTR4(KTR_IW_CXGB, "%s mmid 0x%x mhp %p stag 0x%x", __FUNCTION__, mmid, mhp, stag);
709 static int iwch_dealloc_mw(struct ib_mw *mw)
711 struct iwch_dev *rhp;
715 mhp = to_iwch_mw(mw);
717 mmid = (mw->rkey) >> 8;
718 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
719 remove_handle(rhp, &rhp->mmidr, mmid);
721 CTR4(KTR_IW_CXGB, "%s ib_mw %p mmid 0x%x ptr %p", __FUNCTION__, mw, mmid, mhp);
725 static int iwch_destroy_qp(struct ib_qp *ib_qp)
727 struct iwch_dev *rhp;
729 struct iwch_qp_attributes attrs;
730 struct iwch_ucontext *ucontext;
732 qhp = to_iwch_qp(ib_qp);
735 attrs.next_state = IWCH_QP_STATE_ERROR;
736 iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
737 mtx_lock(&qhp->lock);
739 msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp1", 0);
740 mtx_unlock(&qhp->lock);
742 remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
744 mtx_lock(&qhp->lock);
746 msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp2", 0);
747 mtx_unlock(&qhp->lock);
749 ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
751 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
752 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
754 CTR4(KTR_IW_CXGB, "%s ib_qp %p qpid 0x%0x qhp %p", __FUNCTION__,
755 ib_qp, qhp->wq.qpid, qhp);
760 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
761 struct ib_qp_init_attr *attrs,
762 struct ib_udata *udata)
764 struct iwch_dev *rhp;
767 struct iwch_cq *schp;
768 struct iwch_cq *rchp;
769 struct iwch_create_qp_resp uresp;
770 int wqsize, sqsize, rqsize;
771 struct iwch_ucontext *ucontext;
773 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
774 if (attrs->qp_type != IB_QPT_RC)
775 return ERR_PTR(-EINVAL);
776 php = to_iwch_pd(pd);
778 schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
779 rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
781 return ERR_PTR(-EINVAL);
783 /* The RQT size must be # of entries + 1 rounded up to a power of two */
784 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
785 if (rqsize == attrs->cap.max_recv_wr)
786 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
788 /* T3 doesn't support RQT depth < 16 */
792 if (rqsize > T3_MAX_RQ_SIZE)
793 return ERR_PTR(-EINVAL);
795 if (attrs->cap.max_inline_data > T3_MAX_INLINE)
796 return ERR_PTR(-EINVAL);
799 * NOTE: The SQ and total WQ sizes don't need to be
800 * a power of two. However, all the code assumes
801 * they are. EG: Q_FREECNT() and friends.
803 sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
804 wqsize = roundup_pow_of_two(rqsize + sqsize);
805 CTR4(KTR_IW_CXGB, "%s wqsize %d sqsize %d rqsize %d", __FUNCTION__,
806 wqsize, sqsize, rqsize);
807 qhp = malloc(sizeof(*qhp), M_DEVBUF, M_ZERO|M_NOWAIT);
809 return ERR_PTR(-ENOMEM);
810 qhp->wq.size_log2 = ilog2(wqsize);
811 qhp->wq.rq_size_log2 = ilog2(rqsize);
812 qhp->wq.sq_size_log2 = ilog2(sqsize);
813 ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
814 if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
815 ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
817 return ERR_PTR(-ENOMEM);
820 attrs->cap.max_recv_wr = rqsize - 1;
821 attrs->cap.max_send_wr = sqsize;
822 attrs->cap.max_inline_data = T3_MAX_INLINE;
825 qhp->attr.pd = php->pdid;
826 qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
827 qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
828 qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
829 qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
830 qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
831 qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
832 qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
833 qhp->attr.state = IWCH_QP_STATE_IDLE;
834 qhp->attr.next_state = IWCH_QP_STATE_IDLE;
837 * XXX - These don't get passed in from the openib user
838 * at create time. The CM sets them via a QP modify.
839 * Need to fix... I think the CM should
841 qhp->attr.enable_rdma_read = 1;
842 qhp->attr.enable_rdma_write = 1;
843 qhp->attr.enable_bind = 1;
844 qhp->attr.max_ord = 1;
845 qhp->attr.max_ird = 1;
847 mtx_init(&qhp->lock, "cxgb qp", NULL, MTX_DEF|MTX_DUPOK);
850 if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
851 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
852 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
854 return ERR_PTR(-ENOMEM);
859 struct iwch_mm_entry *mm1, *mm2;
861 mm1 = kmalloc(sizeof *mm1, M_NOWAIT);
863 iwch_destroy_qp(&qhp->ibqp);
864 return ERR_PTR(-ENOMEM);
867 mm2 = kmalloc(sizeof *mm2, M_NOWAIT);
870 iwch_destroy_qp(&qhp->ibqp);
871 return ERR_PTR(-ENOMEM);
874 uresp.qpid = qhp->wq.qpid;
875 uresp.size_log2 = qhp->wq.size_log2;
876 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
877 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
878 mtx_lock(&ucontext->mmap_lock);
879 uresp.key = ucontext->key;
880 ucontext->key += PAGE_SIZE;
881 uresp.db_key = ucontext->key;
882 ucontext->key += PAGE_SIZE;
883 mtx_unlock(&ucontext->mmap_lock);
884 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
887 iwch_destroy_qp(&qhp->ibqp);
888 return ERR_PTR(-EFAULT);
890 mm1->key = uresp.key;
891 mm1->addr = vtophys(qhp->wq.queue);
892 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
893 insert_mmap(ucontext, mm1);
894 mm2->key = uresp.db_key;
895 mm2->addr = qhp->wq.udb & PAGE_MASK;
896 mm2->len = PAGE_SIZE;
897 insert_mmap(ucontext, mm2);
899 qhp->ibqp.qp_num = qhp->wq.qpid;
900 callout_init(&(qhp->timer), TRUE);
901 CTR6(KTR_IW_CXGB, "sq_num_entries %d, rq_num_entries %d "
902 "qpid 0x%0x qhp %p dma_addr 0x%llx size %d",
903 qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
904 qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
905 1 << qhp->wq.size_log2);
909 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
910 int attr_mask, struct ib_udata *udata)
912 struct iwch_dev *rhp;
914 enum iwch_qp_attr_mask mask = 0;
915 struct iwch_qp_attributes attrs;
917 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, ibqp);
919 /* iwarp does not support the RTR state */
920 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
921 attr_mask &= ~IB_QP_STATE;
923 /* Make sure we still have something left to do */
927 memset(&attrs, 0, sizeof attrs);
928 qhp = to_iwch_qp(ibqp);
931 attrs.next_state = iwch_convert_state(attr->qp_state);
932 attrs.enable_rdma_read = (attr->qp_access_flags &
933 IB_ACCESS_REMOTE_READ) ? 1 : 0;
934 attrs.enable_rdma_write = (attr->qp_access_flags &
935 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
936 attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
939 mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
940 mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
941 (IWCH_QP_ATTR_ENABLE_RDMA_READ |
942 IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
943 IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
945 return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
948 void iwch_qp_add_ref(struct ib_qp *qp)
950 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
951 mtx_lock(&to_iwch_qp(qp)->lock);
952 to_iwch_qp(qp)->refcnt++;
953 mtx_unlock(&to_iwch_qp(qp)->lock);
956 void iwch_qp_rem_ref(struct ib_qp *qp)
958 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
959 mtx_lock(&to_iwch_qp(qp)->lock);
960 if (--to_iwch_qp(qp)->refcnt == 0)
961 wakeup(to_iwch_qp(qp));
962 mtx_unlock(&to_iwch_qp(qp)->lock);
965 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
967 CTR3(KTR_IW_CXGB, "%s ib_dev %p qpn 0x%x", __FUNCTION__, dev, qpn);
968 return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
972 static int iwch_query_pkey(struct ib_device *ibdev,
973 u8 port, u16 index, u16 * pkey)
975 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
980 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
981 int index, union ib_gid *gid)
983 struct iwch_dev *dev;
984 struct port_info *pi;
987 CTR5(KTR_IW_CXGB, "%s ibdev %p, port %d, index %d, gid %p",
988 __FUNCTION__, ibdev, port, index, gid);
989 dev = to_iwch_dev(ibdev);
991 PANIC_IF(port == 0 || port > 2);
992 pi = &sc->port[port - 1];
993 memset(&(gid->raw[0]), 0, sizeof(gid->raw));
994 memcpy(&(gid->raw[0]), pi->hw_addr, 6);
998 static int iwch_query_device(struct ib_device *ibdev,
999 struct ib_device_attr *props)
1001 struct iwch_dev *dev;
1004 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1006 dev = to_iwch_dev(ibdev);
1007 sc = dev->rdev.adap;
1008 memset(props, 0, sizeof *props);
1009 memcpy(&props->sys_image_guid, sc->port[0].hw_addr, 6);
1010 props->device_cap_flags = dev->device_cap_flags;
1011 props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1012 props->vendor_id = pci_get_vendor(sc->dev);
1013 props->vendor_part_id = pci_get_device(sc->dev);
1014 props->max_mr_size = dev->attr.max_mr_size;
1015 props->max_qp = dev->attr.max_qps;
1016 props->max_qp_wr = dev->attr.max_wrs;
1017 props->max_sge = dev->attr.max_sge_per_wr;
1018 props->max_sge_rd = 1;
1019 props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1020 props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1021 props->max_cq = dev->attr.max_cqs;
1022 props->max_cqe = dev->attr.max_cqes_per_cq;
1023 props->max_mr = dev->attr.max_mem_regs;
1024 props->max_pd = dev->attr.max_pds;
1025 props->local_ca_ack_delay = 0;
1030 static int iwch_query_port(struct ib_device *ibdev,
1031 u8 port, struct ib_port_attr *props)
1033 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1034 memset(props, 0, sizeof(struct ib_port_attr));
1035 props->max_mtu = IB_MTU_4096;
1036 props->active_mtu = IB_MTU_2048;
1037 props->state = IB_PORT_ACTIVE;
1038 props->port_cap_flags =
1040 IB_PORT_SNMP_TUNNEL_SUP |
1041 IB_PORT_REINIT_SUP |
1042 IB_PORT_DEVICE_MGMT_SUP |
1043 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1044 props->gid_tbl_len = 1;
1045 props->pkey_tbl_len = 1;
1046 props->active_width = 2;
1047 props->active_speed = 2;
1048 props->max_msg_sz = -1;
1053 int iwch_register_device(struct iwch_dev *dev)
1056 struct adapter *sc = dev->rdev.adap;
1058 CTR2(KTR_IW_CXGB, "%s iwch_dev %p", __FUNCTION__, dev);
1059 strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1060 memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1061 memcpy(&dev->ibdev.node_guid, sc->port[0].hw_addr, 6);
1062 dev->device_cap_flags =
1063 (IB_DEVICE_LOCAL_DMA_LKEY |
1064 IB_DEVICE_MEM_WINDOW);
1066 dev->ibdev.uverbs_cmd_mask =
1067 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1068 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1069 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1070 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1071 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1072 (1ull << IB_USER_VERBS_CMD_REG_MR) |
1073 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1074 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1075 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1076 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1077 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1078 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1079 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1080 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1081 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1082 (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1083 (1ull << IB_USER_VERBS_CMD_POST_RECV);
1084 dev->ibdev.node_type = RDMA_NODE_RNIC;
1085 memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1086 dev->ibdev.phys_port_cnt = sc->params.nports;
1087 dev->ibdev.num_comp_vectors = 1;
1088 dev->ibdev.dma_device = dev->rdev.adap->dev;
1089 dev->ibdev.query_device = iwch_query_device;
1090 dev->ibdev.query_port = iwch_query_port;
1091 dev->ibdev.modify_port = iwch_modify_port;
1092 dev->ibdev.query_pkey = iwch_query_pkey;
1093 dev->ibdev.query_gid = iwch_query_gid;
1094 dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1095 dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1096 dev->ibdev.mmap = iwch_mmap;
1097 dev->ibdev.alloc_pd = iwch_allocate_pd;
1098 dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1099 dev->ibdev.create_ah = iwch_ah_create;
1100 dev->ibdev.destroy_ah = iwch_ah_destroy;
1101 dev->ibdev.create_qp = iwch_create_qp;
1102 dev->ibdev.modify_qp = iwch_ib_modify_qp;
1103 dev->ibdev.destroy_qp = iwch_destroy_qp;
1104 dev->ibdev.create_cq = iwch_create_cq;
1105 dev->ibdev.destroy_cq = iwch_destroy_cq;
1106 dev->ibdev.resize_cq = iwch_resize_cq;
1107 dev->ibdev.poll_cq = iwch_poll_cq;
1108 dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1109 dev->ibdev.reg_phys_mr = iwch_register_phys_mem;
1110 dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem;
1111 dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1112 dev->ibdev.dereg_mr = iwch_dereg_mr;
1113 dev->ibdev.alloc_mw = iwch_alloc_mw;
1114 dev->ibdev.bind_mw = iwch_bind_mw;
1115 dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1117 dev->ibdev.attach_mcast = iwch_multicast_attach;
1118 dev->ibdev.detach_mcast = iwch_multicast_detach;
1119 dev->ibdev.process_mad = iwch_process_mad;
1121 dev->ibdev.req_notify_cq = iwch_arm_cq;
1122 dev->ibdev.post_send = iwch_post_send;
1123 dev->ibdev.post_recv = iwch_post_receive;
1124 dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1127 kmalloc(sizeof(struct iw_cm_verbs), M_NOWAIT);
1128 if (!dev->ibdev.iwcm)
1131 dev->ibdev.iwcm->connect = iwch_connect;
1132 dev->ibdev.iwcm->accept = iwch_accept_cr;
1133 dev->ibdev.iwcm->reject = iwch_reject_cr;
1134 dev->ibdev.iwcm->create_listen = iwch_create_listen;
1135 dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1136 dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1137 dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1138 dev->ibdev.iwcm->get_qp = iwch_get_qp;
1140 ret = ib_register_device(&dev->ibdev, NULL);
1147 cxfree(dev->ibdev.iwcm);
1151 void iwch_unregister_device(struct iwch_dev *dev)
1154 ib_unregister_device(&dev->ibdev);
1155 cxfree(dev->ibdev.iwcm);