1 /**************************************************************************
3 Copyright (c) 2007, Chelsio Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/pciio.h>
39 #include <machine/bus.h>
40 #include <machine/resource.h>
41 #include <sys/bus_dma.h>
43 #include <sys/ioccom.h>
45 #include <sys/mutex.h>
46 #include <sys/rwlock.h>
47 #include <sys/linker.h>
48 #include <sys/firmware.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/queue.h>
55 #include <sys/taskqueue.h>
57 #include <sys/queue.h>
59 #include <netinet/in.h>
65 #include <contrib/rdma/ib_verbs.h>
66 #include <contrib/rdma/ib_umem.h>
67 #include <contrib/rdma/ib_user_verbs.h>
69 #include <cxgb_include.h>
70 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
71 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
72 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
73 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
74 #include <ulp/iw_cxgb/iw_cxgb.h>
75 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
76 #include <ulp/iw_cxgb/iw_cxgb_user.h>
79 iwch_modify_port(struct ib_device *ibdev,
80 u8 port, int port_modify_mask,
81 struct ib_port_modify *props)
87 iwch_ah_create(struct ib_pd *pd,
88 struct ib_ah_attr *ah_attr)
90 return ERR_PTR(-ENOSYS);
94 iwch_ah_destroy(struct ib_ah *ah)
99 static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
105 iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
111 iwch_process_mad(struct ib_device *ibdev,
115 struct ib_grh *in_grh,
116 struct ib_mad *in_mad, struct ib_mad *out_mad)
122 iwch_dealloc_ucontext(struct ib_ucontext *context)
124 struct iwch_dev *rhp = to_iwch_dev(context->device);
125 struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
126 struct iwch_mm_entry *mm, *tmp;
128 CTR2(KTR_IW_CXGB, "%s context %p", __FUNCTION__, context);
129 TAILQ_FOREACH_SAFE(mm, &ucontext->mmaps, entry, tmp) {
130 TAILQ_REMOVE(&ucontext->mmaps, mm, entry);
133 cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
138 static struct ib_ucontext *
139 iwch_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
141 struct iwch_ucontext *context;
142 struct iwch_dev *rhp = to_iwch_dev(ibdev);
144 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
145 context = malloc(sizeof(*context), M_DEVBUF, M_ZERO|M_NOWAIT);
147 return ERR_PTR(-ENOMEM);
148 cxio_init_ucontext(&rhp->rdev, &context->uctx);
149 TAILQ_INIT(&context->mmaps);
150 mtx_init(&context->mmap_lock, "ucontext mmap", NULL, MTX_DEF);
151 return &context->ibucontext;
155 iwch_destroy_cq(struct ib_cq *ib_cq)
159 CTR2(KTR_IW_CXGB, "%s ib_cq %p", __FUNCTION__, ib_cq);
160 chp = to_iwch_cq(ib_cq);
162 remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
163 mtx_lock(&chp->lock);
165 msleep(chp, &chp->lock, 0, "iwch_destroy_cq", 0);
166 mtx_unlock(&chp->lock);
168 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
173 static struct ib_cq *
174 iwch_create_cq(struct ib_device *ibdev, int entries, int vector,
175 struct ib_ucontext *ib_context,
176 struct ib_udata *udata)
178 struct iwch_dev *rhp;
180 struct iwch_create_cq_resp uresp;
181 struct iwch_create_cq_req ureq;
182 struct iwch_ucontext *ucontext = NULL;
184 CTR3(KTR_IW_CXGB, "%s ib_dev %p entries %d", __FUNCTION__, ibdev, entries);
185 rhp = to_iwch_dev(ibdev);
186 chp = malloc(sizeof(*chp), M_DEVBUF, M_NOWAIT|M_ZERO);
188 return ERR_PTR(-ENOMEM);
191 ucontext = to_iwch_ucontext(ib_context);
192 if (!t3a_device(rhp)) {
193 if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
195 return ERR_PTR(-EFAULT);
197 chp->user_rptr_addr = (u32 /*__user */*)(unsigned long)ureq.user_rptr_addr;
201 if (t3a_device(rhp)) {
204 * T3A: Add some fluff to handle extra CQEs inserted
205 * for various errors.
206 * Additional CQE possibilities:
208 * incoming RDMA WRITE Failures
209 * incoming RDMA READ REQUEST FAILUREs
210 * NOTE: We cannot ensure the CQ won't overflow.
214 entries = roundup_pow_of_two(entries);
215 chp->cq.size_log2 = ilog2(entries);
217 if (cxio_create_cq(&rhp->rdev, &chp->cq)) {
219 return ERR_PTR(-ENOMEM);
222 chp->ibcq.cqe = 1 << chp->cq.size_log2;
223 mtx_init(&chp->lock, "cxgb cq", NULL, MTX_DEF|MTX_DUPOK);
225 insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid);
228 struct iwch_mm_entry *mm;
230 mm = kmalloc(sizeof *mm, M_NOWAIT);
232 iwch_destroy_cq(&chp->ibcq);
233 return ERR_PTR(-ENOMEM);
235 uresp.cqid = chp->cq.cqid;
236 uresp.size_log2 = chp->cq.size_log2;
237 mtx_lock(&ucontext->mmap_lock);
238 uresp.key = ucontext->key;
239 ucontext->key += PAGE_SIZE;
240 mtx_unlock(&ucontext->mmap_lock);
241 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
243 iwch_destroy_cq(&chp->ibcq);
244 return ERR_PTR(-EFAULT);
247 mm->addr = vtophys(chp->cq.queue);
248 mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
249 sizeof (struct t3_cqe));
250 insert_mmap(ucontext, mm);
252 CTR4(KTR_IW_CXGB, "created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx",
253 chp->cq.cqid, chp, (1 << chp->cq.size_log2),
254 (unsigned long long) chp->cq.dma_addr);
259 iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
262 struct iwch_cq *chp = to_iwch_cq(cq);
263 struct t3_cq oldcq, newcq;
266 CTR3(KTR_IW_CXGB, "%s ib_cq %p cqe %d", __FUNCTION__, cq, cqe);
268 /* We don't downsize... */
272 /* create new t3_cq with new size */
273 cqe = roundup_pow_of_two(cqe+1);
274 newcq.size_log2 = ilog2(cqe);
276 /* Dont allow resize to less than the current wce count */
277 if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
281 /* Quiesce all QPs using this CQ */
282 ret = iwch_quiesce_qps(chp);
287 ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
293 memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
294 sizeof(struct t3_cqe));
296 /* old iwch_qp gets new t3_cq but keeps old cqid */
299 chp->cq.cqid = oldcq.cqid;
301 /* resize new t3_cq to update the HW context */
302 ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
307 chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
309 /* destroy old t3_cq */
310 oldcq.cqid = newcq.cqid;
311 ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
313 log(LOG_ERR, "%s - cxio_destroy_cq failed %d\n",
317 /* add user hooks here */
320 ret = iwch_resume_qps(chp);
328 iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
330 struct iwch_dev *rhp;
332 enum t3_cq_opcode cq_op;
336 chp = to_iwch_cq(ibcq);
338 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
342 if (chp->user_rptr_addr) {
343 if (copyin(&rptr, chp->user_rptr_addr, 4))
345 mtx_lock(&chp->lock);
348 mtx_lock(&chp->lock);
349 CTR2(KTR_IW_CXGB, "%s rptr 0x%x", __FUNCTION__, chp->cq.rptr);
350 err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
351 mtx_unlock(&chp->lock);
353 log(LOG_ERR, "Error %d rearming CQID 0x%x\n", err,
355 if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
362 iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
365 int len = vma->vm_end - vma->vm_start;
366 u32 key = vma->vm_pgoff << PAGE_SHIFT;
367 struct cxio_rdev *rdev_p;
369 struct iwch_mm_entry *mm;
370 struct iwch_ucontext *ucontext;
373 CTR4(KTR_IW_CXGB, "%s pgoff 0x%lx key 0x%x len %d", __FUNCTION__, vma->vm_pgoff,
376 if (vma->vm_start & (PAGE_SIZE-1)) {
380 rdev_p = &(to_iwch_dev(context->device)->rdev);
381 ucontext = to_iwch_ucontext(context);
383 mm = remove_mmap(ucontext, key, len);
389 if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
390 (addr < (rdev_p->rnic_info.udbell_physbase +
391 rdev_p->rnic_info.udbell_len))) {
394 * Map T3 DB register.
396 if (vma->vm_flags & VM_READ) {
400 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
401 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
402 vma->vm_flags &= ~VM_MAYREAD;
403 ret = io_remap_pfn_range(vma, vma->vm_start,
405 len, vma->vm_page_prot);
409 * Map WQ or CQ contig dma memory...
411 ret = remap_pfn_range(vma, vma->vm_start,
413 len, vma->vm_page_prot);
422 static int iwch_deallocate_pd(struct ib_pd *pd)
424 struct iwch_dev *rhp;
427 php = to_iwch_pd(pd);
429 CTR3(KTR_IW_CXGB, "%s ibpd %p pdid 0x%x", __FUNCTION__, pd, php->pdid);
430 cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
435 static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
436 struct ib_ucontext *context,
437 struct ib_udata *udata)
441 struct iwch_dev *rhp;
443 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
444 rhp = (struct iwch_dev *) ibdev;
445 pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
447 return ERR_PTR(-EINVAL);
448 php = malloc(sizeof(*php), M_DEVBUF, M_ZERO|M_NOWAIT);
450 cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
451 return ERR_PTR(-ENOMEM);
456 if (ib_copy_to_udata(udata, &php->pdid, sizeof (__u32))) {
457 iwch_deallocate_pd(&php->ibpd);
458 return ERR_PTR(-EFAULT);
461 CTR3(KTR_IW_CXGB, "%s pdid 0x%0x ptr 0x%p", __FUNCTION__, pdid, php);
465 static int iwch_dereg_mr(struct ib_mr *ib_mr)
467 struct iwch_dev *rhp;
471 CTR2(KTR_IW_CXGB, "%s ib_mr %p", __FUNCTION__, ib_mr);
472 /* There can be no memory windows */
473 if (atomic_load_acq_int(&ib_mr->usecnt))
476 mhp = to_iwch_mr(ib_mr);
478 mmid = mhp->attr.stag >> 8;
479 cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
481 remove_handle(rhp, &rhp->mmidr, mmid);
483 cxfree((void *) (unsigned long) mhp->kva);
485 ib_umem_release(mhp->umem);
486 CTR3(KTR_IW_CXGB, "%s mmid 0x%x ptr %p", __FUNCTION__, mmid, mhp);
491 static struct ib_mr *iwch_register_phys_mem(struct ib_pd *pd,
492 struct ib_phys_buf *buffer_list,
501 struct iwch_dev *rhp;
506 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
507 php = to_iwch_pd(pd);
510 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
512 return ERR_PTR(-ENOMEM);
514 /* First check that we have enough alignment */
515 if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
520 if (num_phys_buf > 1 &&
521 ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
526 ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
527 &total_size, &npages, &shift, &page_list);
532 mhp->attr.pdid = php->pdid;
535 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
536 mhp->attr.va_fbo = *iova_start;
537 mhp->attr.page_size = shift - 12;
539 mhp->attr.len = (u32) total_size;
540 mhp->attr.pbl_size = npages;
541 ret = iwch_register_mem(rhp, php, mhp, shift, page_list);
549 return ERR_PTR(-ret);
553 static int iwch_reregister_phys_mem(struct ib_mr *mr,
556 struct ib_phys_buf *buffer_list,
558 int acc, u64 * iova_start)
561 struct iwch_mr mh, *mhp;
563 struct iwch_dev *rhp;
564 __be64 *page_list = NULL;
570 CTR3(KTR_IW_CXGB, "%s ib_mr %p ib_pd %p", __FUNCTION__, mr, pd);
572 /* There can be no memory windows */
573 if (atomic_load_acq_int(&mr->usecnt))
576 mhp = to_iwch_mr(mr);
578 php = to_iwch_pd(mr->pd);
580 /* make sure we are on the same adapter */
584 memcpy(&mh, mhp, sizeof *mhp);
586 if (mr_rereg_mask & IB_MR_REREG_PD)
587 php = to_iwch_pd(pd);
588 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
589 mh.attr.perms = iwch_ib_to_tpt_access(acc);
590 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
591 ret = build_phys_page_list(buffer_list, num_phys_buf,
593 &total_size, &npages,
599 ret = iwch_reregister_mem(rhp, php, &mh, shift, page_list, npages);
604 if (mr_rereg_mask & IB_MR_REREG_PD)
605 mhp->attr.pdid = php->pdid;
606 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
607 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
608 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
610 mhp->attr.va_fbo = *iova_start;
611 mhp->attr.page_size = shift - 12;
612 mhp->attr.len = (u32) total_size;
613 mhp->attr.pbl_size = npages;
620 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
621 u64 virt, int acc, struct ib_udata *udata)
626 struct ib_umem_chunk *chunk;
627 struct iwch_dev *rhp;
630 struct iwch_reg_user_mr_resp uresp;
635 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
637 php = to_iwch_pd(pd);
639 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_NOWAIT|M_ZERO);
641 return ERR_PTR(-ENOMEM);
643 mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc);
644 if (IS_ERR(mhp->umem)) {
645 err = PTR_ERR(mhp->umem);
647 return ERR_PTR(-err);
650 shift = ffs(mhp->umem->page_size) - 1;
653 TAILQ_FOREACH(chunk, &mhp->umem->chunk_list, entry)
656 pages = kmalloc(n * sizeof(u64), M_NOWAIT);
665 TAILQ_FOREACH(chunk, &mhp->umem->chunk_list, entry)
666 for (j = 0; j < chunk->nmap; ++j) {
667 len = sg_dma_len(&chunk->page_list[j]) >> shift;
668 for (k = 0; k < len; ++k) {
669 pages[i++] = htobe64(sg_dma_address(
670 &chunk->page_list[j]) +
671 mhp->umem->page_size * k);
676 mhp->attr.pdid = php->pdid;
678 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
679 mhp->attr.va_fbo = virt;
680 mhp->attr.page_size = shift - 12;
681 mhp->attr.len = (u32) length;
682 mhp->attr.pbl_size = i;
683 err = iwch_register_mem(rhp, php, mhp, shift, pages);
688 if (udata && !t3a_device(rhp)) {
689 uresp.pbl_addr = (mhp->attr.pbl_addr -
690 rhp->rdev.rnic_info.pbl_base) >> 3;
691 CTR2(KTR_IW_CXGB, "%s user resp pbl_addr 0x%x", __FUNCTION__,
694 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
695 iwch_dereg_mr(&mhp->ibmr);
704 ib_umem_release(mhp->umem);
706 return ERR_PTR(-err);
709 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
711 struct ib_phys_buf bl;
715 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
718 * T3 only supports 32 bits of size.
720 bl.size = 0xffffffff;
723 ibmr = iwch_register_phys_mem(pd, &bl, 1, acc, &kva);
727 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd)
729 struct iwch_dev *rhp;
736 php = to_iwch_pd(pd);
738 mhp = malloc(sizeof(*mhp), M_DEVBUF, M_ZERO|M_NOWAIT);
740 return ERR_PTR(-ENOMEM);
741 ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
744 return ERR_PTR(-ret);
747 mhp->attr.pdid = php->pdid;
748 mhp->attr.type = TPT_MW;
749 mhp->attr.stag = stag;
751 insert_handle(rhp, &rhp->mmidr, mhp, mmid);
752 CTR4(KTR_IW_CXGB, "%s mmid 0x%x mhp %p stag 0x%x", __FUNCTION__, mmid, mhp, stag);
756 static int iwch_dealloc_mw(struct ib_mw *mw)
758 struct iwch_dev *rhp;
762 mhp = to_iwch_mw(mw);
764 mmid = (mw->rkey) >> 8;
765 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
766 remove_handle(rhp, &rhp->mmidr, mmid);
768 CTR4(KTR_IW_CXGB, "%s ib_mw %p mmid 0x%x ptr %p", __FUNCTION__, mw, mmid, mhp);
772 static int iwch_destroy_qp(struct ib_qp *ib_qp)
774 struct iwch_dev *rhp;
776 struct iwch_qp_attributes attrs;
777 struct iwch_ucontext *ucontext;
779 qhp = to_iwch_qp(ib_qp);
782 attrs.next_state = IWCH_QP_STATE_ERROR;
783 iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
784 mtx_lock(&qhp->lock);
786 msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp1", 0);
787 mtx_unlock(&qhp->lock);
789 remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
791 mtx_lock(&qhp->lock);
793 msleep(qhp, &qhp->lock, 0, "iwch_destroy_qp2", 0);
794 mtx_unlock(&qhp->lock);
796 ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
798 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
799 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
801 CTR4(KTR_IW_CXGB, "%s ib_qp %p qpid 0x%0x qhp %p", __FUNCTION__,
802 ib_qp, qhp->wq.qpid, qhp);
807 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
808 struct ib_qp_init_attr *attrs,
809 struct ib_udata *udata)
811 struct iwch_dev *rhp;
814 struct iwch_cq *schp;
815 struct iwch_cq *rchp;
816 struct iwch_create_qp_resp uresp;
817 int wqsize, sqsize, rqsize;
818 struct iwch_ucontext *ucontext;
820 CTR2(KTR_IW_CXGB, "%s ib_pd %p", __FUNCTION__, pd);
821 if (attrs->qp_type != IB_QPT_RC)
822 return ERR_PTR(-EINVAL);
823 php = to_iwch_pd(pd);
825 schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
826 rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
828 return ERR_PTR(-EINVAL);
830 /* The RQT size must be # of entries + 1 rounded up to a power of two */
831 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
832 if (rqsize == attrs->cap.max_recv_wr)
833 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
835 /* T3 doesn't support RQT depth < 16 */
839 if (rqsize > T3_MAX_RQ_SIZE)
840 return ERR_PTR(-EINVAL);
842 if (attrs->cap.max_inline_data > T3_MAX_INLINE)
843 return ERR_PTR(-EINVAL);
846 * NOTE: The SQ and total WQ sizes don't need to be
847 * a power of two. However, all the code assumes
848 * they are. EG: Q_FREECNT() and friends.
850 sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
851 wqsize = roundup_pow_of_two(rqsize + sqsize);
852 CTR4(KTR_IW_CXGB, "%s wqsize %d sqsize %d rqsize %d", __FUNCTION__,
853 wqsize, sqsize, rqsize);
854 qhp = malloc(sizeof(*qhp), M_DEVBUF, M_ZERO|M_NOWAIT);
856 return ERR_PTR(-ENOMEM);
857 qhp->wq.size_log2 = ilog2(wqsize);
858 qhp->wq.rq_size_log2 = ilog2(rqsize);
859 qhp->wq.sq_size_log2 = ilog2(sqsize);
860 ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
861 if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
862 ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
864 return ERR_PTR(-ENOMEM);
867 attrs->cap.max_recv_wr = rqsize - 1;
868 attrs->cap.max_send_wr = sqsize;
869 attrs->cap.max_inline_data = T3_MAX_INLINE;
872 qhp->attr.pd = php->pdid;
873 qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
874 qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
875 qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
876 qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
877 qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
878 qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
879 qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
880 qhp->attr.state = IWCH_QP_STATE_IDLE;
881 qhp->attr.next_state = IWCH_QP_STATE_IDLE;
884 * XXX - These don't get passed in from the openib user
885 * at create time. The CM sets them via a QP modify.
886 * Need to fix... I think the CM should
888 qhp->attr.enable_rdma_read = 1;
889 qhp->attr.enable_rdma_write = 1;
890 qhp->attr.enable_bind = 1;
891 qhp->attr.max_ord = 1;
892 qhp->attr.max_ird = 1;
894 mtx_init(&qhp->lock, "cxgb qp", NULL, MTX_DEF|MTX_DUPOK);
896 insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid);
900 struct iwch_mm_entry *mm1, *mm2;
902 mm1 = kmalloc(sizeof *mm1, M_NOWAIT);
904 iwch_destroy_qp(&qhp->ibqp);
905 return ERR_PTR(-ENOMEM);
908 mm2 = kmalloc(sizeof *mm2, M_NOWAIT);
911 iwch_destroy_qp(&qhp->ibqp);
912 return ERR_PTR(-ENOMEM);
915 uresp.qpid = qhp->wq.qpid;
916 uresp.size_log2 = qhp->wq.size_log2;
917 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
918 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
919 mtx_lock(&ucontext->mmap_lock);
920 uresp.key = ucontext->key;
921 ucontext->key += PAGE_SIZE;
922 uresp.db_key = ucontext->key;
923 ucontext->key += PAGE_SIZE;
924 mtx_unlock(&ucontext->mmap_lock);
925 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
928 iwch_destroy_qp(&qhp->ibqp);
929 return ERR_PTR(-EFAULT);
931 mm1->key = uresp.key;
932 mm1->addr = vtophys(qhp->wq.queue);
933 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
934 insert_mmap(ucontext, mm1);
935 mm2->key = uresp.db_key;
936 mm2->addr = qhp->wq.udb & PAGE_MASK;
937 mm2->len = PAGE_SIZE;
938 insert_mmap(ucontext, mm2);
940 qhp->ibqp.qp_num = qhp->wq.qpid;
941 callout_init(&(qhp->timer), TRUE);
942 CTR6(KTR_IW_CXGB, "sq_num_entries %d, rq_num_entries %d "
943 "qpid 0x%0x qhp %p dma_addr 0x%llx size %d",
944 qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
945 qhp->wq.qpid, qhp, (unsigned long long) qhp->wq.dma_addr,
946 1 << qhp->wq.size_log2);
950 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
951 int attr_mask, struct ib_udata *udata)
953 struct iwch_dev *rhp;
955 enum iwch_qp_attr_mask mask = 0;
956 struct iwch_qp_attributes attrs;
958 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, ibqp);
960 /* iwarp does not support the RTR state */
961 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
962 attr_mask &= ~IB_QP_STATE;
964 /* Make sure we still have something left to do */
968 memset(&attrs, 0, sizeof attrs);
969 qhp = to_iwch_qp(ibqp);
972 attrs.next_state = iwch_convert_state(attr->qp_state);
973 attrs.enable_rdma_read = (attr->qp_access_flags &
974 IB_ACCESS_REMOTE_READ) ? 1 : 0;
975 attrs.enable_rdma_write = (attr->qp_access_flags &
976 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
977 attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
980 mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
981 mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
982 (IWCH_QP_ATTR_ENABLE_RDMA_READ |
983 IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
984 IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
986 return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
989 void iwch_qp_add_ref(struct ib_qp *qp)
991 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
992 mtx_lock(&to_iwch_qp(qp)->lock);
993 to_iwch_qp(qp)->refcnt++;
994 mtx_unlock(&to_iwch_qp(qp)->lock);
997 void iwch_qp_rem_ref(struct ib_qp *qp)
999 CTR2(KTR_IW_CXGB, "%s ib_qp %p", __FUNCTION__, qp);
1000 mtx_lock(&to_iwch_qp(qp)->lock);
1001 if (--to_iwch_qp(qp)->refcnt == 0)
1002 wakeup(to_iwch_qp(qp));
1003 mtx_unlock(&to_iwch_qp(qp)->lock);
1006 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
1008 CTR3(KTR_IW_CXGB, "%s ib_dev %p qpn 0x%x", __FUNCTION__, dev, qpn);
1009 return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
1013 static int iwch_query_pkey(struct ib_device *ibdev,
1014 u8 port, u16 index, u16 * pkey)
1016 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1021 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
1022 int index, union ib_gid *gid)
1024 struct iwch_dev *dev;
1025 struct port_info *pi;
1027 CTR5(KTR_IW_CXGB, "%s ibdev %p, port %d, index %d, gid %p",
1028 __FUNCTION__, ibdev, port, index, gid);
1029 dev = to_iwch_dev(ibdev);
1030 PANIC_IF(port == 0 || port > 2);
1031 pi = ((struct port_info *)dev->rdev.port_info.lldevs[port-1]->if_softc);
1032 memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1033 memcpy(&(gid->raw[0]), pi->hw_addr, 6);
1037 static int iwch_query_device(struct ib_device *ibdev,
1038 struct ib_device_attr *props)
1041 struct iwch_dev *dev;
1042 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1044 dev = to_iwch_dev(ibdev);
1045 memset(props, 0, sizeof *props);
1047 memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->if_addr.ifa_addr, 6);
1049 props->device_cap_flags = dev->device_cap_flags;
1051 props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1052 props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1054 props->max_mr_size = ~0ull;
1055 props->max_qp = dev->attr.max_qps;
1056 props->max_qp_wr = dev->attr.max_wrs;
1057 props->max_sge = dev->attr.max_sge_per_wr;
1058 props->max_sge_rd = 1;
1059 props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1060 props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1061 props->max_cq = dev->attr.max_cqs;
1062 props->max_cqe = dev->attr.max_cqes_per_cq;
1063 props->max_mr = dev->attr.max_mem_regs;
1064 props->max_pd = dev->attr.max_pds;
1065 props->local_ca_ack_delay = 0;
1070 static int iwch_query_port(struct ib_device *ibdev,
1071 u8 port, struct ib_port_attr *props)
1073 CTR2(KTR_IW_CXGB, "%s ibdev %p", __FUNCTION__, ibdev);
1074 props->max_mtu = IB_MTU_4096;
1079 props->state = IB_PORT_ACTIVE;
1080 props->phys_state = 0;
1081 props->port_cap_flags =
1083 IB_PORT_SNMP_TUNNEL_SUP |
1084 IB_PORT_REINIT_SUP |
1085 IB_PORT_DEVICE_MGMT_SUP |
1086 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1087 props->gid_tbl_len = 1;
1088 props->pkey_tbl_len = 1;
1089 props->qkey_viol_cntr = 0;
1090 props->active_width = 2;
1091 props->active_speed = 2;
1092 props->max_msg_sz = -1;
1098 static ssize_t show_rev(struct class_device *cdev, char *buf)
1100 struct iwch_dev *dev = container_of(cdev, struct iwch_dev,
1102 CTR2(KTR_IW_CXGB, "%s class dev 0x%p", __FUNCTION__, cdev);
1103 return sprintf(buf, "%d\n", dev->rdev.t3cdev_p->type);
1106 static ssize_t show_fw_ver(struct class_device *cdev, char *buf)
1108 struct iwch_dev *dev = container_of(cdev, struct iwch_dev,
1110 struct ethtool_drvinfo info;
1111 struct net_device *lldev = dev->rdev.t3cdev_p->lldev;
1113 CTR2(KTR_IW_CXGB, "%s class dev 0x%p", __FUNCTION__, cdev);
1114 lldev->ethtool_ops->get_drvinfo(lldev, &info);
1115 return sprintf(buf, "%s\n", info.fw_version);
1118 static ssize_t show_hca(struct class_device *cdev, char *buf)
1120 struct iwch_dev *dev = container_of(cdev, struct iwch_dev,
1122 struct ethtool_drvinfo info;
1123 struct net_device *lldev = dev->rdev.t3cdev_p->lldev;
1125 CTR2(KTR_IW_CXGB, "%s class dev 0x%p", __FUNCTION__, cdev);
1126 lldev->ethtool_ops->get_drvinfo(lldev, &info);
1127 return sprintf(buf, "%s\n", info.driver);
1130 static ssize_t show_board(struct class_device *cdev, char *buf)
1132 struct iwch_dev *dev = container_of(cdev, struct iwch_dev,
1134 CTR2(KTR_IW_CXGB, "%s class dev 0x%p", __FUNCTION__, dev);
1136 return sprintf(buf, "%x.%x\n", dev->rdev.rnic_info.pdev->vendor,
1137 dev->rdev.rnic_info.pdev->device);
1139 return sprintf(buf, "%x.%x\n", 0xdead, 0xbeef); /* XXX */
1143 static CLASS_DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1144 static CLASS_DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
1145 static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1146 static CLASS_DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
1148 static struct class_device_attribute *iwch_class_attributes[] = {
1149 &class_device_attr_hw_rev,
1150 &class_device_attr_fw_ver,
1151 &class_device_attr_hca_type,
1152 &class_device_attr_board_id
1156 int iwch_register_device(struct iwch_dev *dev)
1162 CTR2(KTR_IW_CXGB, "%s iwch_dev %p", __FUNCTION__, dev);
1163 strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1164 memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1166 memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1168 dev->device_cap_flags =
1169 (IB_DEVICE_ZERO_STAG |
1170 IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
1172 dev->ibdev.uverbs_cmd_mask =
1173 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1174 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1175 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1176 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1177 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1178 (1ull << IB_USER_VERBS_CMD_REG_MR) |
1179 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1180 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1181 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1182 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1183 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1184 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1185 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1186 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1187 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1188 (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1189 (1ull << IB_USER_VERBS_CMD_POST_RECV);
1190 dev->ibdev.node_type = RDMA_NODE_RNIC;
1191 memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1192 dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1193 dev->ibdev.num_comp_vectors = 1;
1194 dev->ibdev.dma_device = dev->rdev.rnic_info.pdev;
1195 dev->ibdev.query_device = iwch_query_device;
1196 dev->ibdev.query_port = iwch_query_port;
1197 dev->ibdev.modify_port = iwch_modify_port;
1198 dev->ibdev.query_pkey = iwch_query_pkey;
1199 dev->ibdev.query_gid = iwch_query_gid;
1200 dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1201 dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1203 dev->ibdev.mmap = iwch_mmap;
1205 dev->ibdev.alloc_pd = iwch_allocate_pd;
1206 dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1207 dev->ibdev.create_ah = iwch_ah_create;
1208 dev->ibdev.destroy_ah = iwch_ah_destroy;
1209 dev->ibdev.create_qp = iwch_create_qp;
1210 dev->ibdev.modify_qp = iwch_ib_modify_qp;
1211 dev->ibdev.destroy_qp = iwch_destroy_qp;
1212 dev->ibdev.create_cq = iwch_create_cq;
1213 dev->ibdev.destroy_cq = iwch_destroy_cq;
1214 dev->ibdev.resize_cq = iwch_resize_cq;
1215 dev->ibdev.poll_cq = iwch_poll_cq;
1216 dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1217 dev->ibdev.reg_phys_mr = iwch_register_phys_mem;
1218 dev->ibdev.rereg_phys_mr = iwch_reregister_phys_mem;
1219 dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1220 dev->ibdev.dereg_mr = iwch_dereg_mr;
1221 dev->ibdev.alloc_mw = iwch_alloc_mw;
1222 dev->ibdev.bind_mw = iwch_bind_mw;
1223 dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1225 dev->ibdev.attach_mcast = iwch_multicast_attach;
1226 dev->ibdev.detach_mcast = iwch_multicast_detach;
1227 dev->ibdev.process_mad = iwch_process_mad;
1229 dev->ibdev.req_notify_cq = iwch_arm_cq;
1230 dev->ibdev.post_send = iwch_post_send;
1231 dev->ibdev.post_recv = iwch_post_receive;
1235 (struct iw_cm_verbs *) kmalloc(sizeof(struct iw_cm_verbs),
1237 dev->ibdev.iwcm->connect = iwch_connect;
1238 dev->ibdev.iwcm->accept = iwch_accept_cr;
1239 dev->ibdev.iwcm->reject = iwch_reject_cr;
1240 dev->ibdev.iwcm->create_listen = iwch_create_listen;
1241 dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1242 dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1243 dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1244 dev->ibdev.iwcm->get_qp = iwch_get_qp;
1246 ret = ib_register_device(&dev->ibdev);
1250 for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i) {
1251 ret = class_device_create_file(&dev->ibdev.class_dev,
1252 iwch_class_attributes[i]);
1262 ib_unregister_device(&dev->ibdev);
1267 void iwch_unregister_device(struct iwch_dev *dev)
1272 CTR2(KTR_IW_CXGB, "%s iwch_dev %p", __FUNCTION__, dev);
1274 for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i)
1275 class_device_remove_file(&dev->ibdev.class_dev,
1276 iwch_class_attributes[i]);
1278 ib_unregister_device(&dev->ibdev);