2 * Copyright (c) 2018-2019 Cavium, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
34 #include "qlnxr_def.h"
35 #include "rdma_common.h"
36 #include "qlnxr_roce.h"
39 #define upper_32_bits(x) (uint32_t)(x >> 32)
40 #define lower_32_bits(x) (uint32_t)(x)
41 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
43 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \
45 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
46 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
49 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
51 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
52 (sge)->length = cpu_to_le32(vlength); \
53 (sge)->flags = cpu_to_le32(vflags); \
56 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
58 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \
59 (hdr)->num_sges = num_sge; \
62 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
64 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
65 (sge)->length = cpu_to_le32(vlength); \
66 (sge)->l_key = cpu_to_le32(vlkey); \
69 #define NIPQUAD(addr) \
70 ((unsigned char *)&addr)[0], \
71 ((unsigned char *)&addr)[1], \
72 ((unsigned char *)&addr)[2], \
73 ((unsigned char *)&addr)[3]
76 qlnxr_check_srq_params(struct ib_pd *ibpd,
77 struct qlnxr_dev *dev,
78 struct ib_srq_init_attr *attrs);
81 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
82 struct qlnxr_srq *srq,
83 struct qlnxr_create_srq_ureq *ureq,
84 int access, int dmasync);
87 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
88 struct qlnxr_dev *dev,
89 struct ib_srq_init_attr *init_attr);
92 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
93 struct qlnxr_srq *srq,
94 struct ib_udata *udata);
97 qlnxr_free_srq_user_params(struct qlnxr_srq *srq);
100 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq);
103 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq);
106 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index,
109 struct qlnxr_dev *dev;
112 dev = get_qlnxr_dev(ibdev);
115 QL_DPRINT12(ha, "enter\n");
117 memset(sgid->raw, 0, sizeof(sgid->raw));
119 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));
121 QL_DPRINT12(ha, "exit\n");
127 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
130 struct qlnxr_dev *dev;
133 dev = get_qlnxr_dev(ibdev);
135 QL_DPRINT12(ha, "enter index: %d\n", index);
138 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */
139 //if (!rdma_cap_roce_gid_table(ibdev, port)) {
140 if (!(rdma_protocol_roce(ibdev, port) &&
141 ibdev->add_gid && ibdev->del_gid)) {
142 QL_DPRINT11(ha, "acquire gid failed\n");
146 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
147 if (ret == -EAGAIN) {
148 memcpy(sgid, &zgid, sizeof(*sgid));
152 if ((index >= QLNXR_MAX_SGID) || (index < 0)) {
153 QL_DPRINT12(ha, "invalid gid index %d\n", index);
154 memset(sgid, 0, sizeof(*sgid));
157 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
159 QL_DPRINT12(ha, "exit : %p\n", sgid);
165 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
166 struct ib_udata *udata)
168 struct qlnxr_dev *dev;
170 struct ecore_rdma_destroy_srq_in_params destroy_in_params;
171 struct ecore_rdma_create_srq_out_params out_params;
172 struct ecore_rdma_create_srq_in_params in_params;
173 u64 pbl_base_addr, phy_prod_pair_addr;
174 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
175 struct ib_ucontext *ib_ctx = NULL;
176 struct qlnxr_srq_hwq_info *hw_srq;
177 struct qlnxr_ucontext *ctx = NULL;
178 struct qlnxr_create_srq_ureq ureq;
179 u32 page_cnt, page_size;
180 struct qlnxr_srq *srq;
183 dev = get_qlnxr_dev((ibpd->device));
186 QL_DPRINT12(ha, "enter\n");
188 ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
190 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
192 QL_DPRINT11(ha, "cannot allocate memory for srq\n");
193 return NULL; //@@@ : TODO what to return here?
197 hw_srq = &srq->hw_srq;
198 spin_lock_init(&srq->lock);
199 memset(&in_params, 0, sizeof(in_params));
201 if (udata && ibpd->uobject && ibpd->uobject->context) {
202 ib_ctx = ibpd->uobject->context;
203 ctx = get_qlnxr_ucontext(ib_ctx);
205 memset(&ureq, 0, sizeof(ureq));
206 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
208 QL_DPRINT11(ha, "problem"
209 " copying data from user space\n");
213 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
217 page_cnt = srq->usrq.pbl_info.num_pbes;
218 pbl_base_addr = srq->usrq.pbl_tbl->pa;
219 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
220 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT
221 // page_size = BIT(srq->usrq.umem->page_shift);
223 page_size = srq->usrq.umem->page_size;
225 struct ecore_chain *pbl;
226 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr);
231 page_cnt = ecore_chain_get_page_cnt(pbl);
232 pbl_base_addr = ecore_chain_get_pbl_phys(pbl);
233 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
234 page_size = pbl->elem_per_page << 4;
237 in_params.pd_id = pd->pd_id;
238 in_params.pbl_base_addr = pbl_base_addr;
239 in_params.prod_pair_addr = phy_prod_pair_addr;
240 in_params.num_pages = page_cnt;
241 in_params.page_size = page_size;
243 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
247 srq->srq_id = out_params.srq_id;
250 ret = qlnxr_copy_srq_uresp(dev, srq, udata);
255 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
258 memset(&in_params, 0, sizeof(in_params));
259 destroy_in_params.srq_id = srq->srq_id;
260 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
264 qlnxr_free_srq_user_params(srq);
266 qlnxr_free_srq_kernel_params(srq);
270 return ERR_PTR(-EFAULT);
274 qlnxr_destroy_srq(struct ib_srq *ibsrq)
276 struct qlnxr_dev *dev;
277 struct qlnxr_srq *srq;
279 struct ecore_rdma_destroy_srq_in_params in_params;
281 srq = get_qlnxr_srq(ibsrq);
285 memset(&in_params, 0, sizeof(in_params));
286 in_params.srq_id = srq->srq_id;
288 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);
290 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)
291 qlnxr_free_srq_user_params(srq);
293 qlnxr_free_srq_kernel_params(srq);
295 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
301 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
302 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
304 struct qlnxr_dev *dev;
305 struct qlnxr_srq *srq;
307 struct ecore_rdma_modify_srq_in_params in_params;
310 srq = get_qlnxr_srq(ibsrq);
314 QL_DPRINT12(ha, "enter\n");
315 if (attr_mask & IB_SRQ_MAX_WR) {
316 QL_DPRINT12(ha, "invalid attribute mask=0x%x"
317 " specified for %p\n", attr_mask, srq);
321 if (attr_mask & IB_SRQ_LIMIT) {
322 if (attr->srq_limit >= srq->hw_srq.max_wr) {
323 QL_DPRINT12(ha, "invalid srq_limit=0x%x"
324 " (max_srq_limit = 0x%x)\n",
325 attr->srq_limit, srq->hw_srq.max_wr);
328 memset(&in_params, 0, sizeof(in_params));
329 in_params.srq_id = srq->srq_id;
330 in_params.wqe_limit = attr->srq_limit;
331 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);
336 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);
341 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
343 struct qlnxr_dev *dev;
344 struct qlnxr_srq *srq;
346 struct ecore_rdma_device *qattr;
347 srq = get_qlnxr_srq(ibsrq);
350 //qattr = &dev->attr;
351 qattr = ecore_rdma_query_device(dev->rdma_ctx);
352 QL_DPRINT12(ha, "enter\n");
354 if (!dev->rdma_ctx) {
355 QL_DPRINT12(ha, "called with invalid params"
356 " rdma_ctx is NULL\n");
360 srq_attr->srq_limit = qattr->max_srq;
361 srq_attr->max_wr = qattr->max_srq_wr;
362 srq_attr->max_sge = qattr->max_sge;
364 QL_DPRINT12(ha, "exit\n");
368 /* Increment srq wr producer by one */
370 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info)
375 /* Increment srq wr consumer by one */
377 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info)
382 /* get_port_immutable verb is not available in FreeBSD */
385 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
386 struct ib_port_immutable *immutable)
388 struct qlnxr_dev *dev;
390 dev = get_qlnxr_dev(ibdev);
393 QL_DPRINT12(ha, "entered but not implemented!!!\n");
398 qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
399 struct ib_recv_wr **bad_wr)
401 struct qlnxr_dev *dev;
402 struct qlnxr_srq *srq;
404 struct qlnxr_srq_hwq_info *hw_srq;
405 struct ecore_chain *pbl;
410 srq = get_qlnxr_srq(ibsrq);
413 hw_srq = &srq->hw_srq;
415 QL_DPRINT12(ha, "enter\n");
416 spin_lock_irqsave(&srq->lock, flags);
418 pbl = &srq->hw_srq.pbl;
420 struct rdma_srq_wqe_header *hdr;
423 if (!qlnxr_srq_elem_left(hw_srq) ||
424 wr->num_sge > srq->hw_srq.max_sges) {
425 QL_DPRINT11(ha, "WR cannot be posted"
426 " (%d, %d) || (%d > %d)\n",
427 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
428 wr->num_sge, srq->hw_srq.max_sges);
434 hdr = ecore_chain_produce(pbl);
435 num_sge = wr->num_sge;
436 /* Set number of sge and WR id in header */
437 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
439 /* PBL is maintained in case of WR granularity.
440 * So increment WR producer in case we post a WR.
442 qlnxr_inc_srq_wr_prod(hw_srq);
446 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n",
447 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
449 for (i = 0; i < wr->num_sge; i++) {
450 struct rdma_srq_sge *srq_sge =
451 ecore_chain_produce(pbl);
452 /* Set SGE length, lkey and address */
453 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
454 wr->sg_list[i].length, wr->sg_list[i].lkey);
456 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",
457 i, srq_sge->length, srq_sge->l_key,
458 srq_sge->addr.hi, srq_sge->addr.lo);
463 * SRQ prod is 8 bytes. Need to update SGE prod in index
464 * in first 4 bytes and need to update WQE prod in next
467 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;
468 offset = offsetof(struct rdma_srq_producers, wqe_prod);
469 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
471 /* Flush prod after updating it */
476 QL_DPRINT12(ha, "Elements in SRQ: %d\n",
477 ecore_chain_get_elem_left(pbl));
479 spin_unlock_irqrestore(&srq->lock, flags);
480 QL_DPRINT12(ha, "exit\n");
485 #if __FreeBSD_version < 1102000
486 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
488 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
489 struct ib_udata *udata)
490 #endif /* #if __FreeBSD_version < 1102000 */
493 struct qlnxr_dev *dev;
494 struct ecore_rdma_device *qattr;
497 dev = get_qlnxr_dev(ibdev);
500 QL_DPRINT12(ha, "enter\n");
502 #if __FreeBSD_version > 1102000
503 if (udata->inlen || udata->outlen)
505 #endif /* #if __FreeBSD_version > 1102000 */
507 if (dev->rdma_ctx == NULL) {
511 qattr = ecore_rdma_query_device(dev->rdma_ctx);
513 memset(attr, 0, sizeof *attr);
515 attr->fw_ver = qattr->fw_ver;
516 attr->sys_image_guid = qattr->sys_image_guid;
517 attr->max_mr_size = qattr->max_mr_size;
518 attr->page_size_cap = qattr->page_size_caps;
519 attr->vendor_id = qattr->vendor_id;
520 attr->vendor_part_id = qattr->vendor_part_id;
521 attr->hw_ver = qattr->hw_ver;
522 attr->max_qp = qattr->max_qp;
523 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
524 IB_DEVICE_RC_RNR_NAK_GEN |
525 IB_DEVICE_LOCAL_DMA_LKEY |
526 IB_DEVICE_MEM_MGT_EXTENSIONS;
528 attr->max_sge = qattr->max_sge;
529 attr->max_sge_rd = qattr->max_sge;
530 attr->max_cq = qattr->max_cq;
531 attr->max_cqe = qattr->max_cqe;
532 attr->max_mr = qattr->max_mr;
533 attr->max_mw = qattr->max_mw;
534 attr->max_pd = qattr->max_pd;
535 attr->atomic_cap = dev->atomic_cap;
536 attr->max_fmr = qattr->max_fmr;
537 attr->max_map_per_fmr = 16; /* TBD: FMR */
539 /* There is an implicit assumption in some of the ib_xxx apps that the
540 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in
541 * communication the qp_rd_atom is passed to the other side and used as
542 * init_rd_atom without check device capabilities for init_rd_atom.
543 * for this reason, we set the qp_rd_atom to be the minimum between the
544 * two...There is an additional assumption in mlx4 driver that the
545 * values are power of two, fls is performed on the value - 1, which
546 * in fact gives a larger power of two for values which are not a power
547 * of two. This should be fixed in mlx4 driver, but until then ->
548 * we provide a value that is a power of two in our code.
550 attr->max_qp_init_rd_atom =
551 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
552 attr->max_qp_rd_atom =
553 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
554 attr->max_qp_init_rd_atom);
556 attr->max_srq = qattr->max_srq;
557 attr->max_srq_sge = qattr->max_srq_sge;
558 attr->max_srq_wr = qattr->max_srq_wr;
560 /* TODO: R&D to more properly configure the following */
561 attr->local_ca_ack_delay = qattr->dev_ack_delay;
562 attr->max_fast_reg_page_list_len = qattr->max_mr/8;
563 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;
564 attr->max_ah = qattr->max_ah;
566 QL_DPRINT12(ha, "exit\n");
571 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width)
575 *ib_speed = IB_SPEED_SDR;
576 *ib_width = IB_WIDTH_1X;
579 *ib_speed = IB_SPEED_QDR;
580 *ib_width = IB_WIDTH_1X;
584 *ib_speed = IB_SPEED_DDR;
585 *ib_width = IB_WIDTH_4X;
589 *ib_speed = IB_SPEED_EDR;
590 *ib_width = IB_WIDTH_1X;
594 *ib_speed = IB_SPEED_QDR;
595 *ib_width = IB_WIDTH_4X;
599 *ib_speed = IB_SPEED_QDR;
600 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50...
604 *ib_speed = IB_SPEED_EDR;
605 *ib_width = IB_WIDTH_4X;
610 *ib_speed = IB_SPEED_SDR;
611 *ib_width = IB_WIDTH_1X;
617 qlnxr_query_port(struct ib_device *ibdev, uint8_t port,
618 struct ib_port_attr *attr)
620 struct qlnxr_dev *dev;
621 struct ecore_rdma_port *rdma_port;
624 dev = get_qlnxr_dev(ibdev);
627 QL_DPRINT12(ha, "enter\n");
630 QL_DPRINT12(ha, "port [%d] > 1 \n", port);
634 if (dev->rdma_ctx == NULL) {
635 QL_DPRINT12(ha, "rdma_ctx == NULL\n");
639 rdma_port = ecore_rdma_query_port(dev->rdma_ctx);
640 memset(attr, 0, sizeof *attr);
642 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {
643 attr->state = IB_PORT_ACTIVE;
644 attr->phys_state = 5;
646 attr->state = IB_PORT_DOWN;
647 attr->phys_state = 3;
650 attr->max_mtu = IB_MTU_4096;
651 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu);
656 attr->port_cap_flags = 0;
658 if (QLNX_IS_IWARP(dev)) {
659 attr->gid_tbl_len = 1;
660 attr->pkey_tbl_len = 1;
662 attr->gid_tbl_len = QLNXR_MAX_SGID;
663 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;
666 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
667 attr->qkey_viol_cntr = 0;
669 get_link_speed_and_width(rdma_port->link_speed,
670 &attr->active_speed, &attr->active_width);
672 attr->max_msg_sz = rdma_port->max_msg_size;
673 attr->max_vl_num = 4; /* TODO -> figure this one out... */
675 QL_DPRINT12(ha, "state = %d phys_state = %d "
676 " link_speed = %d active_speed = %d active_width = %d"
677 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"
678 " max_msg_sz = 0x%x max_vl_num = 0x%x \n",
679 attr->state, attr->phys_state,
680 rdma_port->link_speed, attr->active_speed,
681 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,
682 attr->max_msg_sz, attr->max_vl_num);
684 QL_DPRINT12(ha, "exit\n");
689 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask,
690 struct ib_port_modify *props)
692 struct qlnxr_dev *dev;
695 dev = get_qlnxr_dev(ibdev);
698 QL_DPRINT12(ha, "enter\n");
701 QL_DPRINT12(ha, "port (%d) > 1\n", port);
705 QL_DPRINT12(ha, "exit\n");
710 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
712 struct qlnxr_dev *dev;
715 dev = get_qlnxr_dev(ibdev);
718 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);
720 return IB_LINK_LAYER_ETHERNET;
724 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
725 struct ib_udata *udata)
727 struct qlnxr_pd *pd = NULL;
730 struct qlnxr_dev *dev;
733 dev = get_qlnxr_dev(ibdev);
736 QL_DPRINT12(ha, "ibdev = %p context = %p"
737 " udata = %p enter\n", ibdev, context, udata);
739 if (dev->rdma_ctx == NULL) {
740 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
745 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
748 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
752 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
754 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
760 if (udata && context) {
761 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
763 QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
764 ecore_rdma_free_pd(dev->rdma_ctx, pd_id);
768 pd->uctx = get_qlnxr_ucontext(context);
772 atomic_add_rel_32(&dev->pd_count, 1);
773 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
774 pd, pd_id, dev->pd_count);
780 QL_DPRINT12(ha, "exit -1\n");
785 qlnxr_dealloc_pd(struct ib_pd *ibpd)
788 struct qlnxr_dev *dev;
791 pd = get_qlnxr_pd(ibpd);
792 dev = get_qlnxr_dev((ibpd->device));
795 QL_DPRINT12(ha, "enter\n");
798 QL_DPRINT11(ha, "pd = NULL\n");
800 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
802 atomic_subtract_rel_32(&dev->pd_count, 1);
803 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
804 pd, pd->pd_id, dev->pd_count);
807 QL_DPRINT12(ha, "exit\n");
811 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge)
812 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */
813 /* Should be part of HSI */
814 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */
815 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
817 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *);
818 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *);
821 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
823 struct qlnxr_dev *dev;
826 dev = get_qlnxr_dev(ibdev);
829 QL_DPRINT12(ha, "enter index = 0x%x\n", index);
831 if (index > QLNXR_ROCE_PKEY_TABLE_LEN)
834 *pkey = QLNXR_ROCE_PKEY_DEFAULT;
836 QL_DPRINT12(ha, "exit\n");
841 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask,
846 QL_DPRINT12(ha, "enter \n");
850 #if __FreeBSD_version >= 1100000
853 #if __FreeBSD_version >= 1102000
856 dgid = &attr->ah_attr.grh.dgid;
857 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
859 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) {
860 *vlan_id = tmp_vlan_id;
864 tmp_vlan_id = attr->vlan_id;
866 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) {
867 *vlan_id = tmp_vlan_id;
871 #endif /* #if __FreeBSD_version > 1102000 */
876 #endif /* #if __FreeBSD_version >= 1100000 */
878 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);
884 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr,
886 struct qlnxr_dev *dev,
888 struct ecore_rdma_modify_qp_in_params *qp_params)
895 QL_DPRINT12(ha, "enter\n");
897 memcpy(&qp_params->sgid.bytes[0],
898 &dev->sgid_tbl[qp->sgid_idx].raw[0],
899 sizeof(qp_params->sgid.bytes));
900 memcpy(&qp_params->dgid.bytes[0],
901 &attr->ah_attr.grh.dgid.raw[0],
902 sizeof(qp_params->dgid));
904 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);
906 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {
907 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
908 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
911 QL_DPRINT12(ha, "exit\n");
916 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
923 QL_DPRINT12(ha, "enter\n");
925 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
927 QL_DPRINT11(ha, "mm = NULL\n");
931 mm->key.phy_addr = phy_addr;
933 /* This function might be called with a length which is not a multiple
934 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
935 * forces this granularity by increasing the requested size if needed.
936 * When qedr_mmap is called, it will search the list with the updated
937 * length as a key. To prevent search failures, the length is rounded up
938 * in advance to PAGE_SIZE.
940 mm->key.len = roundup(len, PAGE_SIZE);
941 INIT_LIST_HEAD(&mm->entry);
943 mutex_lock(&uctx->mm_list_lock);
944 list_add(&mm->entry, &uctx->mm_head);
945 mutex_unlock(&uctx->mm_list_lock);
947 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
948 (unsigned long long)mm->key.phy_addr,
949 (unsigned long)mm->key.len, uctx);
955 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
963 QL_DPRINT12(ha, "enter\n");
965 mutex_lock(&uctx->mm_list_lock);
966 list_for_each_entry(mm, &uctx->mm_head, entry) {
967 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
973 mutex_unlock(&uctx->mm_list_lock);
976 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",
977 mm->key.phy_addr, mm->key.len, uctx, found);
983 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
984 struct ib_udata *udata)
987 struct qlnxr_ucontext *ctx;
988 struct qlnxr_alloc_ucontext_resp uresp;
989 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
990 qlnx_host_t *ha = dev->ha;
991 struct ecore_rdma_add_user_out_params oparams;
994 return ERR_PTR(-EFAULT);
997 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
999 return ERR_PTR(-ENOMEM);
1001 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
1004 "Failed to allocate a DPI for a new RoCE application "
1005 ",rc = %d. To overcome this, consider to increase "
1006 "the number of DPIs, increase the doorbell BAR size "
1007 "or just close unnecessary RoCE applications. In "
1008 "order to increase the number of DPIs consult the "
1013 ctx->dpi = oparams.dpi;
1014 ctx->dpi_addr = oparams.dpi_addr;
1015 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
1016 ctx->dpi_size = oparams.dpi_size;
1017 INIT_LIST_HEAD(&ctx->mm_head);
1018 mutex_init(&ctx->mm_list_lock);
1020 memset(&uresp, 0, sizeof(uresp));
1021 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled)
1022 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out
1023 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled)
1024 < udata->outlen ? 1 : 0; //TODO: figure this out
1025 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count)
1026 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out
1027 uresp.db_pa = ctx->dpi_phys_addr;
1028 uresp.db_size = ctx->dpi_size;
1029 uresp.max_send_wr = dev->attr.max_sqe;
1030 uresp.max_recv_wr = dev->attr.max_rqe;
1031 uresp.max_srq_wr = dev->attr.max_srq_wr;
1032 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
1033 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
1034 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
1035 uresp.max_cqes = QLNXR_MAX_CQES;
1037 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1043 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
1046 QL_DPRINT12(ha, "Allocated user context %p\n",
1049 return &ctx->ibucontext;
1056 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
1058 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
1059 struct qlnxr_dev *dev = uctx->dev;
1060 qlnx_host_t *ha = dev->ha;
1061 struct qlnxr_mm *mm, *tmp;
1064 QL_DPRINT12(ha, "Deallocating user context %p\n",
1068 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
1071 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
1072 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"
1074 mm->key.phy_addr, mm->key.len, uctx);
1075 list_del(&mm->entry);
1083 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
1085 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context);
1086 struct qlnxr_dev *dev = get_qlnxr_dev((context->device));
1087 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
1089 unsigned long len = (vma->vm_end - vma->vm_start);
1096 #if __FreeBSD_version > 1102000
1097 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
1099 unmapped_db = dev->db_phys_addr;
1100 #endif /* #if __FreeBSD_version > 1102000 */
1102 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"
1103 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
1104 vm_page, vma->vm_pgoff, unmapped_db,
1107 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
1108 QL_DPRINT11(ha, "Vma_start not page aligned "
1109 "vm_start = %ld vma_end = %ld\n", vma->vm_start,
1114 found = qlnxr_search_mmap(ucontext, vm_page, len);
1116 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n",
1121 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1123 #if __FreeBSD_version > 1102000
1125 if ((vm_page < unmapped_db) ||
1126 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {
1127 QL_DPRINT11(ha, "failed pages are outside of dpi;"
1128 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",
1129 vm_page, unmapped_db, ucontext->dpi_size);
1133 if (vma->vm_flags & VM_READ) {
1134 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n");
1138 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1139 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
1144 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
1146 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1148 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1150 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1151 PAGE_SIZE, vma->vm_page_prot);
1153 QL_DPRINT12(ha, "Mapping chains\n");
1154 rc = io_remap_pfn_range(vma, vma->vm_start,
1155 vma->vm_pgoff, len, vma->vm_page_prot);
1158 #endif /* #if __FreeBSD_version > 1102000 */
1160 QL_DPRINT12(ha, "exit [%d]\n", rc);
1165 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc)
1167 struct qlnxr_mr *mr;
1168 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1169 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
1175 QL_DPRINT12(ha, "enter\n");
1177 if (acc & IB_ACCESS_MW_BIND) {
1178 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n");
1181 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1184 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);
1188 mr->type = QLNXR_MR_DMA;
1190 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1192 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);
1196 /* index only, 18 bit long, lkey = itid << 8 | key */
1197 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1198 mr->hw_mr.pd = pd->pd_id;
1199 mr->hw_mr.local_read = 1;
1200 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1201 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1202 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1203 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1204 mr->hw_mr.dma_mr = true;
1206 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1208 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);
1212 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1214 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1215 mr->hw_mr.remote_atomic) {
1216 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1219 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);
1224 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1228 QL_DPRINT12(ha, "exit [%d]\n", rc);
1234 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info,
1235 struct qlnxr_pbl *pbl)
1242 QL_DPRINT12(ha, "enter\n");
1244 for (i = 0; i < pbl_info->num_pbls; i++) {
1247 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,
1248 pbl_info->pbl_size);
1252 QL_DPRINT12(ha, "exit\n");
1256 #define MIN_FW_PBL_PAGE_SIZE (4*1024)
1257 #define MAX_FW_PBL_PAGE_SIZE (64*1024)
1259 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
1260 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
1261 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE)
1263 static struct qlnxr_pbl *
1264 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev,
1265 struct qlnxr_pbl_info *pbl_info, gfp_t flags)
1269 dma_addr_t *pbl_main_tbl;
1270 struct qlnxr_pbl *pbl_table;
1276 QL_DPRINT12(ha, "enter\n");
1278 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);
1281 QL_DPRINT12(ha, "pbl_table = NULL\n");
1285 for (i = 0; i < pbl_info->num_pbls; i++) {
1286 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);
1288 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i);
1292 memset(va, 0, pbl_info->pbl_size);
1293 pbl_table[i].va = va;
1294 pbl_table[i].pa = pa;
1297 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
1298 * the first one with physical pointers to all of the rest
1300 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
1301 for (i = 0; i < pbl_info->num_pbls - 1; i++)
1302 pbl_main_tbl[i] = pbl_table[i + 1].pa;
1304 QL_DPRINT12(ha, "exit\n");
1308 qlnxr_free_pbl(dev, pbl_info, pbl_table);
1310 QL_DPRINT12(ha, "exit with error\n");
1315 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev,
1316 struct qlnxr_pbl_info *pbl_info,
1318 int two_layer_capable)
1327 QL_DPRINT12(ha, "enter\n");
1329 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
1330 if (num_pbes > MAX_PBES_TWO_LAYER) {
1331 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n",
1336 /* calculate required pbl page size */
1337 pbl_size = MIN_FW_PBL_PAGE_SIZE;
1338 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
1339 NUM_PBES_ON_PAGE(pbl_size);
1341 while (pbl_capacity < num_pbes) {
1343 pbl_capacity = pbl_size / sizeof(u64);
1344 pbl_capacity = pbl_capacity * pbl_capacity;
1347 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
1348 num_pbls++; /* One for the layer0 ( points to the pbls) */
1349 pbl_info->two_layered = true;
1351 /* One layered PBL */
1353 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \
1354 roundup_pow_of_two((num_pbes * sizeof(u64))));
1355 pbl_info->two_layered = false;
1358 pbl_info->num_pbls = num_pbls;
1359 pbl_info->pbl_size = pbl_size;
1360 pbl_info->num_pbes = num_pbes;
1362 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n",
1363 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
1368 #define upper_32_bits(x) (uint32_t)(x >> 32)
1369 #define lower_32_bits(x) (uint32_t)(x)
1372 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem,
1373 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
1375 struct regpair *pbe;
1376 struct qlnxr_pbl *pbl_tbl;
1377 struct scatterlist *sg;
1378 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
1381 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1383 struct ib_umem_chunk *chunk = NULL;
1390 QL_DPRINT12(ha, "enter\n");
1393 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
1397 if (!pbl_info->num_pbes) {
1398 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
1402 /* If we have a two layered pbl, the first pbl points to the rest
1403 * of the pbls and the first entry lays on the second pbl in the table
1405 if (pbl_info->two_layered)
1410 pbe = (struct regpair *)pbl_tbl->va;
1412 QL_DPRINT12(ha, "pbe is NULL\n");
1418 shift = ilog2(umem->page_size);
1420 #ifndef DEFINE_IB_UMEM_WITH_CHUNK
1422 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1424 list_for_each_entry(chunk, &umem->chunk_list, list) {
1425 /* get all the dma regions from the chunk. */
1426 for (i = 0; i < chunk->nmap; i++) {
1427 sg = &chunk->page_list[i];
1429 pages = sg_dma_len(sg) >> shift;
1430 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
1431 /* store the page address in pbe */
1433 cpu_to_le32(sg_dma_address(sg) +
1434 (umem->page_size * pg_cnt));
1436 cpu_to_le32(upper_32_bits
1437 ((sg_dma_address(sg) +
1438 umem->page_size * pg_cnt)));
1441 "Populate pbl table:"
1442 " pbe->addr=0x%x:0x%x "
1443 " pbe_cnt = %d total_num_pbes=%d"
1444 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
1445 total_num_pbes, pbe);
1451 if (total_num_pbes == pbl_info->num_pbes)
1454 /* if the given pbl is full storing the pbes,
1458 (pbl_info->pbl_size / sizeof(u64))) {
1460 pbe = (struct regpair *)pbl_tbl->va;
1464 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1468 QL_DPRINT12(ha, "exit\n");
1473 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info)
1475 struct qlnxr_pbl *pbl, *tmp;
1480 QL_DPRINT12(ha, "enter\n");
1482 if (info->pbl_table)
1483 list_add_tail(&info->pbl_table->list_entry,
1484 &info->free_pbl_list);
1486 if (!list_empty(&info->inuse_pbl_list))
1487 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
1489 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
1490 list_del(&pbl->list_entry);
1491 qlnxr_free_pbl(dev, &info->pbl_info, pbl);
1493 QL_DPRINT12(ha, "exit\n");
1499 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info,
1500 size_t page_list_len, bool two_layered)
1503 struct qlnxr_pbl *tmp;
1508 QL_DPRINT12(ha, "enter\n");
1510 INIT_LIST_HEAD(&info->free_pbl_list);
1511 INIT_LIST_HEAD(&info->inuse_pbl_list);
1513 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,
1514 page_list_len, two_layered);
1516 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);
1520 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1522 if (!info->pbl_table) {
1524 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n");
1528 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);
1530 /* in usual case we use 2 PBLs, so we add one to free
1531 * list and allocating another one
1533 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1536 QL_DPRINT11(ha, "Extra PBL is not allocated\n");
1537 goto done; /* it's OK if second allocation fails, so rc = 0*/
1540 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
1542 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);
1546 free_mr_info(dev, info);
1548 QL_DPRINT12(ha, "exit [%d]\n", rc);
1554 #if __FreeBSD_version >= 1102000
1555 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1556 u64 usr_addr, int acc, struct ib_udata *udata)
1558 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1559 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id)
1560 #endif /* #if __FreeBSD_version >= 1102000 */
1563 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1564 struct qlnxr_mr *mr;
1565 struct qlnxr_pd *pd;
1570 QL_DPRINT12(ha, "enter\n");
1572 pd = get_qlnxr_pd(ibpd);
1574 QL_DPRINT12(ha, "qedr_register user mr pd = %d"
1575 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
1576 pd->pd_id, start, len, usr_addr, acc);
1578 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
1580 "(acc & IB_ACCESS_REMOTE_WRITE &&"
1581 " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
1582 return ERR_PTR(-EINVAL);
1585 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1587 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
1591 mr->type = QLNXR_MR_USER;
1593 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
1594 if (IS_ERR(mr->umem)) {
1596 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);
1600 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
1603 "qlnxr_init_mr_info failed [%d]\n", rc);
1607 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
1608 &mr->info.pbl_info);
1610 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1613 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
1617 /* index only, 18 bit long, lkey = itid << 8 | key */
1618 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1620 mr->hw_mr.pd = pd->pd_id;
1621 mr->hw_mr.local_read = 1;
1622 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1623 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1624 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1625 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1626 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
1627 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
1628 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
1629 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
1630 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */
1632 #if __FreeBSD_version >= 1102000
1633 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
1635 mr->hw_mr.fbo = mr->umem->offset;
1637 mr->hw_mr.length = len;
1638 mr->hw_mr.vaddr = usr_addr;
1639 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
1640 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
1641 mr->hw_mr.dma_mr = false;
1643 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1645 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
1649 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1650 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1651 mr->hw_mr.remote_atomic)
1652 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1654 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);
1659 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1661 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1665 QL_DPRINT12(ha, "exit [%d]\n", rc);
1666 return (ERR_PTR(rc));
1670 qlnxr_dereg_mr(struct ib_mr *ib_mr)
1672 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr);
1673 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
1679 QL_DPRINT12(ha, "enter\n");
1681 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))
1682 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1684 /* it could be user registered memory. */
1686 ib_umem_release(mr->umem);
1692 QL_DPRINT12(ha, "exit\n");
1697 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev,
1698 struct qlnxr_cq *cq, struct ib_udata *udata)
1700 struct qlnxr_create_cq_uresp uresp;
1706 QL_DPRINT12(ha, "enter\n");
1708 memset(&uresp, 0, sizeof(uresp));
1710 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1711 uresp.icid = cq->icid;
1713 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1716 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",
1720 QL_DPRINT12(ha, "exit [%d]\n", rc);
1725 consume_cqe(struct qlnxr_cq *cq)
1728 if (cq->latest_cqe == cq->toggle_cqe)
1729 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1731 cq->latest_cqe = ecore_chain_consume(&cq->pbl);
1735 qlnxr_align_cq_entries(int entries)
1737 u64 size, aligned_size;
1739 /* We allocate an extra entry that we don't report to the FW.
1741 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes
1742 * (for performance purposes). Allocating an extra entry and telling
1743 * the FW we have less prevents overwriting the first entry in case of
1744 * a wrap i.e. when the FW writes the last entry and the application
1745 * hasn't read the first one.
1747 size = (entries + 1) * QLNXR_CQE_SIZE;
1749 /* We align to PAGE_SIZE.
1751 * Since the CQ is going to be mapped and the mapping is anyhow in whole
1752 * kernel pages we benefit from the possibly extra CQEs.
1754 aligned_size = ALIGN(size, PAGE_SIZE);
1756 /* note: for CQs created in user space the result of this function
1757 * should match the size mapped in user space
1759 return (aligned_size / QLNXR_CQE_SIZE);
1763 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev,
1764 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len,
1765 int access, int dmasync, int alloc_and_init)
1773 QL_DPRINT12(ha, "enter\n");
1775 q->buf_addr = buf_addr;
1776 q->buf_len = buf_len;
1778 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"
1779 " dmasync : %x\n", q->buf_addr, q->buf_len,
1782 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
1784 if (IS_ERR(q->umem)) {
1785 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));
1786 return PTR_ERR(q->umem);
1789 page_cnt = ib_umem_page_count(q->umem);
1790 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,
1791 0 /* SQ and RQ don't support dual layer pbl.
1792 * CQ may, but this is yet uncoded.
1795 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);
1799 if (alloc_and_init) {
1800 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
1803 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1808 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
1810 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
1813 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1819 QL_DPRINT12(ha, "exit\n");
1823 ib_umem_release(q->umem);
1826 QL_DPRINT12(ha, "exit [%d]\n", rc);
1830 #if __FreeBSD_version >= 1102000
1833 qlnxr_create_cq(struct ib_device *ibdev,
1834 const struct ib_cq_init_attr *attr,
1835 struct ib_ucontext *ib_ctx,
1836 struct ib_udata *udata)
1840 #if __FreeBSD_version >= 1100000
1843 qlnxr_create_cq(struct ib_device *ibdev,
1844 struct ib_cq_init_attr *attr,
1845 struct ib_ucontext *ib_ctx,
1846 struct ib_udata *udata)
1851 qlnxr_create_cq(struct ib_device *ibdev,
1854 struct ib_ucontext *ib_ctx,
1855 struct ib_udata *udata)
1856 #endif /* #if __FreeBSD_version >= 1100000 */
1858 #endif /* #if __FreeBSD_version >= 1102000 */
1860 struct qlnxr_ucontext *ctx;
1861 struct ecore_rdma_destroy_cq_out_params destroy_oparams;
1862 struct ecore_rdma_destroy_cq_in_params destroy_iparams;
1863 struct qlnxr_dev *dev;
1864 struct ecore_rdma_create_cq_in_params params;
1865 struct qlnxr_create_cq_ureq ureq;
1867 #if __FreeBSD_version >= 1100000
1868 int vector = attr->comp_vector;
1869 int entries = attr->cqe;
1871 struct qlnxr_cq *cq;
1872 int chain_entries, rc, page_cnt;
1877 dev = get_qlnxr_dev(ibdev);
1880 QL_DPRINT12(ha, "called from %s. entries = %d, "
1882 (udata ? "User Lib" : "Kernel"), entries, vector);
1884 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params));
1885 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params));
1886 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params));
1888 if (entries > QLNXR_MAX_CQES) {
1890 "the number of entries %d is too high. "
1891 "Must be equal or below %d.\n",
1892 entries, QLNXR_MAX_CQES);
1893 return ERR_PTR(-EINVAL);
1895 chain_entries = qlnxr_align_cq_entries(entries);
1896 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
1898 cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
1901 return ERR_PTR(-ENOMEM);
1904 memset(&ureq, 0, sizeof(ureq));
1906 if (ib_copy_from_udata(&ureq, udata,
1907 min(sizeof(ureq), udata->inlen))) {
1908 QL_DPRINT11(ha, "ib_copy_from_udata failed\n");
1913 QL_DPRINT11(ha, "ureq.len == 0\n");
1917 cq->cq_type = QLNXR_CQ_TYPE_USER;
1919 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
1920 IB_ACCESS_LOCAL_WRITE, 1, 1);
1922 pbl_ptr = cq->q.pbl_tbl->pa;
1923 page_cnt = cq->q.pbl_info.num_pbes;
1924 cq->ibcq.cqe = chain_entries;
1926 cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
1928 rc = ecore_chain_alloc(&dev->ha->cdev,
1929 ECORE_CHAIN_USE_TO_CONSUME,
1930 ECORE_CHAIN_MODE_PBL,
1931 ECORE_CHAIN_CNT_TYPE_U32,
1933 sizeof(union roce_cqe),
1939 page_cnt = ecore_chain_get_page_cnt(&cq->pbl);
1940 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);
1941 cq->ibcq.cqe = cq->pbl.capacity;
1944 params.cq_handle_hi = upper_32_bits((uintptr_t)cq);
1945 params.cq_handle_lo = lower_32_bits((uintptr_t)cq);
1946 params.cnq_id = vector;
1947 params.cq_size = chain_entries - 1;
1948 params.pbl_num_pages = page_cnt;
1949 params.pbl_ptr = pbl_ptr;
1950 params.pbl_two_level = 0;
1952 if (ib_ctx != NULL) {
1953 ctx = get_qlnxr_ucontext(ib_ctx);
1954 params.dpi = ctx->dpi;
1956 params.dpi = dev->dpi;
1959 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
1964 cq->sig = QLNXR_CQ_MAGIC_NUMBER;
1965 spin_lock_init(&cq->cq_lock);
1968 rc = qlnxr_copy_cq_uresp(dev, cq, udata);
1972 /* Generate doorbell address.
1973 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.
1974 * TODO: consider moving to device scope as it is a function of
1976 * TODO: add ifdef if plan to support 16 bit.
1978 cq->db_addr = dev->db_addr +
1979 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1980 cq->db.data.icid = cq->icid;
1981 cq->db.data.params = DB_AGG_CMD_SET <<
1982 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1984 /* point to the very last element, passing it we will toggle */
1985 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);
1986 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1988 /* must be different from pbl_toggle */
1989 cq->latest_cqe = NULL;
1991 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
1994 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"
1995 " number of entries = 0x%x\n",
1996 cq->icid, cq, params.cq_size);
1997 QL_DPRINT12(ha,"cq_addr = %p\n", cq);
2001 destroy_iparams.icid = cq->icid;
2002 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);
2005 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2007 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2010 ib_umem_release(cq->q.umem);
2014 QL_DPRINT12(ha, "exit error\n");
2016 return ERR_PTR(-EINVAL);
2019 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
2022 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2027 QL_DPRINT12(ha, "enter/exit\n");
2033 qlnxr_destroy_cq(struct ib_cq *ibcq)
2035 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2036 struct ecore_rdma_destroy_cq_out_params oparams;
2037 struct ecore_rdma_destroy_cq_in_params iparams;
2038 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
2044 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);
2048 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation
2049 * that all completions with notification are dealt with. The rest
2050 * of the completions are not interesting
2053 /* GSIs CQs are handled by driver, so they don't exist in the FW */
2055 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {
2056 iparams.icid = cq->icid;
2058 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
2061 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
2066 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
2067 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2070 if (ibcq->uobject && ibcq->uobject->context) {
2071 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2072 ib_umem_release(cq->q.umem);
2079 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
2085 qlnxr_check_qp_attrs(struct ib_pd *ibpd,
2086 struct qlnxr_dev *dev,
2087 struct ib_qp_init_attr *attrs,
2088 struct ib_udata *udata)
2090 struct ecore_rdma_device *qattr;
2093 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2096 QL_DPRINT12(ha, "enter\n");
2098 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);
2099 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);
2100 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);
2102 #if __FreeBSD_version < 1102000
2103 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type);
2106 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);
2107 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);
2108 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);
2109 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);
2110 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);
2111 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",
2112 attrs->cap.max_inline_data);
2114 #if __FreeBSD_version < 1102000
2115 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n",
2116 attrs->cap.qpg_tss_mask_sz);
2119 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);
2120 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);
2121 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);
2122 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);
2123 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);
2124 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",
2125 (void *)qattr->sys_image_guid);
2126 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);
2127 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);
2128 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);
2129 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);
2130 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);
2131 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);
2132 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",
2133 qattr->max_qp_resp_rd_atomic_resc);
2134 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",
2135 qattr->max_qp_req_rd_atomic_resc);
2136 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",
2137 qattr->max_dev_resp_rd_atomic_resc);
2138 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);
2139 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);
2140 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);
2141 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);
2142 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);
2143 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);
2144 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);
2145 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);
2146 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",
2147 qattr->max_mr_mw_fmr_pbl);
2148 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",
2149 (void *)qattr->max_mr_mw_fmr_size);
2150 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);
2151 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);
2152 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);
2153 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);
2154 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",
2155 qattr->max_stats_queues);
2156 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);
2157 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",
2158 (void *)qattr->page_size_caps);
2159 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);
2160 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);
2161 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",
2162 qattr->bad_pkey_counter);
2164 if ((attrs->qp_type == IB_QPT_GSI) && udata) {
2165 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n");
2169 if (udata && !(ibpd->uobject && ibpd->uobject->context)) {
2170 QL_DPRINT12(ha, "called from user without context\n");
2174 /* QP0... attrs->qp_type == IB_QPT_GSI */
2175 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
2176 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n",
2180 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {
2181 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n");
2184 /* Skip the check for QP1 to support CM size of 128 */
2185 if (attrs->cap.max_send_wr > qattr->max_wqe) {
2186 QL_DPRINT12(ha, "cannot create a SQ with %d elements "
2187 " (max_send_wr=0x%x)\n",
2188 attrs->cap.max_send_wr, qattr->max_wqe);
2191 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {
2192 QL_DPRINT12(ha, "cannot create a RQ with %d elements"
2193 " (max_recv_wr=0x%x)\n",
2194 attrs->cap.max_recv_wr, qattr->max_wqe);
2197 if (attrs->cap.max_inline_data > qattr->max_inline) {
2199 "unsupported inline data size=0x%x "
2200 "requested (max_inline=0x%x)\n",
2201 attrs->cap.max_inline_data, qattr->max_inline);
2204 if (attrs->cap.max_send_sge > qattr->max_sge) {
2206 "unsupported send_sge=0x%x "
2207 "requested (max_send_sge=0x%x)\n",
2208 attrs->cap.max_send_sge, qattr->max_sge);
2211 if (attrs->cap.max_recv_sge > qattr->max_sge) {
2213 "unsupported recv_sge=0x%x requested "
2214 " (max_recv_sge=0x%x)\n",
2215 attrs->cap.max_recv_sge, qattr->max_sge);
2218 /* unprivileged user space cannot create special QP */
2219 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
2221 "userspace can't create special QPs of type=0x%x\n",
2225 /* allow creating only one GSI type of QP */
2226 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
2228 "create qp: GSI special QPs already created.\n");
2232 /* verify consumer QPs are not trying to use GSI QP's CQ */
2233 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
2234 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);
2235 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);
2237 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||
2238 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {
2239 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n");
2243 QL_DPRINT12(ha, "exit\n");
2248 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
2249 struct qlnxr_srq *srq,
2250 struct ib_udata *udata)
2252 struct qlnxr_create_srq_uresp uresp;
2258 QL_DPRINT12(ha, "enter\n");
2260 memset(&uresp, 0, sizeof(uresp));
2262 uresp.srq_id = srq->srq_id;
2264 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2266 QL_DPRINT12(ha, "exit [%d]\n", rc);
2271 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev,
2272 struct qlnxr_create_qp_uresp *uresp,
2273 struct qlnxr_qp *qp)
2279 /* Return if QP is associated with SRQ instead of RQ */
2280 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);
2285 /* iWARP requires two doorbells per RQ. */
2286 if (QLNX_IS_IWARP(dev)) {
2287 uresp->rq_db_offset =
2288 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2289 uresp->rq_db2_offset =
2290 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2292 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "
2293 "uresp->rq_db2_offset = 0x%x\n",
2294 uresp->rq_db_offset, uresp->rq_db2_offset);
2296 uresp->rq_db_offset =
2297 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2299 uresp->rq_icid = qp->icid;
2301 QL_DPRINT12(ha, "exit\n");
2306 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev,
2307 struct qlnxr_create_qp_uresp *uresp,
2308 struct qlnxr_qp *qp)
2314 QL_DPRINT12(ha, "enter\n");
2316 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2318 /* iWARP uses the same cid for rq and sq*/
2319 if (QLNX_IS_IWARP(dev)) {
2320 uresp->sq_icid = qp->icid;
2321 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);
2323 uresp->sq_icid = qp->icid + 1;
2325 QL_DPRINT12(ha, "exit\n");
2330 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev,
2331 struct qlnxr_qp *qp,
2332 struct ib_udata *udata)
2335 struct qlnxr_create_qp_uresp uresp;
2340 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);
2342 memset(&uresp, 0, sizeof(uresp));
2343 qlnxr_copy_sq_uresp(dev, &uresp, qp);
2344 qlnxr_copy_rq_uresp(dev, &uresp, qp);
2346 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
2347 uresp.qp_id = qp->qp_id;
2349 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2351 QL_DPRINT12(ha, "exit [%d]\n", rc);
2356 qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
2357 struct qlnxr_qp *qp,
2358 struct qlnxr_pd *pd,
2359 struct ib_qp_init_attr *attrs)
2365 QL_DPRINT12(ha, "enter\n");
2367 spin_lock_init(&qp->q_lock);
2369 atomic_set(&qp->refcnt, 1);
2371 qp->sig = QLNXR_QP_MAGIC_NUMBER;
2372 qp->qp_type = attrs->qp_type;
2373 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2374 qp->sq.max_sges = attrs->cap.max_send_sge;
2375 qp->state = ECORE_ROCE_QP_STATE_RESET;
2376 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
2377 qp->sq_cq = get_qlnxr_cq(attrs->send_cq);
2378 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);
2382 /* QP is associated with RQ instead of SRQ */
2383 qp->rq.max_sges = attrs->cap.max_recv_sge;
2384 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
2385 qp->rq.max_sges, qp->rq_cq->icid);
2387 qp->srq = get_qlnxr_srq(attrs->srq);
2391 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d,"
2392 " state = %d, signaled = %d, use_srq=%d\n",
2393 pd->pd_id, qp->qp_type, qp->max_inline_data,
2394 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));
2395 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
2396 qp->sq.max_sges, qp->sq_cq->icid);
2401 qlnxr_check_srq_params(struct ib_pd *ibpd,
2402 struct qlnxr_dev *dev,
2403 struct ib_srq_init_attr *attrs)
2405 struct ecore_rdma_device *qattr;
2409 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2411 QL_DPRINT12(ha, "enter\n");
2413 if (attrs->attr.max_wr > qattr->max_srq_wqe) {
2414 QL_DPRINT12(ha, "unsupported srq_wr=0x%x"
2415 " requested (max_srq_wr=0x%x)\n",
2416 attrs->attr.max_wr, qattr->max_srq_wr);
2420 if (attrs->attr.max_sge > qattr->max_sge) {
2422 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
2423 attrs->attr.max_sge, qattr->max_sge);
2427 if (attrs->attr.srq_limit > attrs->attr.max_wr) {
2429 "unsupported srq_limit=0x%x requested"
2430 " (max_srq_limit=0x%x)\n",
2431 attrs->attr.srq_limit, attrs->attr.srq_limit);
2435 QL_DPRINT12(ha, "exit\n");
2440 qlnxr_free_srq_user_params(struct qlnxr_srq *srq)
2442 struct qlnxr_dev *dev = srq->dev;
2447 QL_DPRINT12(ha, "enter\n");
2449 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2450 ib_umem_release(srq->usrq.umem);
2451 ib_umem_release(srq->prod_umem);
2453 QL_DPRINT12(ha, "exit\n");
2458 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq)
2460 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2461 struct qlnxr_dev *dev = srq->dev;
2466 QL_DPRINT12(ha, "enter\n");
2468 ecore_chain_free(dev->cdev, &hw_srq->pbl);
2470 qlnx_dma_free_coherent(&dev->cdev,
2471 hw_srq->virt_prod_pair_addr,
2472 hw_srq->phy_prod_pair_addr,
2473 sizeof(struct rdma_srq_producers));
2475 QL_DPRINT12(ha, "exit\n");
2481 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
2482 struct qlnxr_srq *srq,
2483 struct qlnxr_create_srq_ureq *ureq,
2484 int access, int dmasync)
2486 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2487 struct ib_umem_chunk *chunk;
2489 struct scatterlist *sg;
2491 struct qlnxr_dev *dev = srq->dev;
2496 QL_DPRINT12(ha, "enter\n");
2498 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
2499 ureq->srq_len, access, dmasync, 1);
2503 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
2504 sizeof(struct rdma_srq_producers),
2506 if (IS_ERR(srq->prod_umem)) {
2507 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2508 ib_umem_release(srq->usrq.umem);
2510 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n",
2511 PTR_ERR(srq->prod_umem));
2513 return PTR_ERR(srq->prod_umem);
2516 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2517 chunk = container_of((&srq->prod_umem->chunk_list)->next,
2518 typeof(*chunk), list);
2519 sg = &chunk->page_list[0];
2521 sg = srq->prod_umem->sg_head.sgl;
2523 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
2525 QL_DPRINT12(ha, "exit\n");
2530 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
2531 struct qlnxr_dev *dev,
2532 struct ib_srq_init_attr *init_attr)
2534 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2535 dma_addr_t phy_prod_pair_addr;
2536 u32 num_elems, max_wr;
2543 QL_DPRINT12(ha, "enter\n");
2545 va = qlnx_dma_alloc_coherent(&dev->cdev,
2546 &phy_prod_pair_addr,
2547 sizeof(struct rdma_srq_producers));
2549 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n");
2553 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
2554 hw_srq->virt_prod_pair_addr = va;
2556 max_wr = init_attr->attr.max_wr;
2558 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE;
2560 rc = ecore_chain_alloc(dev->cdev,
2561 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2562 ECORE_CHAIN_MODE_PBL,
2563 ECORE_CHAIN_CNT_TYPE_U32,
2565 ECORE_RDMA_SRQ_WQE_ELEM_SIZE,
2566 &hw_srq->pbl, NULL);
2569 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);
2573 hw_srq->max_wr = max_wr;
2574 hw_srq->num_elems = num_elems;
2575 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;
2577 QL_DPRINT12(ha, "exit\n");
2581 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,
2582 sizeof(struct rdma_srq_producers));
2584 QL_DPRINT12(ha, "exit [%d]\n", rc);
2589 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev,
2590 struct qlnxr_pd *pd,
2591 struct qlnxr_qp *qp,
2592 struct ib_qp_init_attr *attrs,
2593 bool fmr_and_reserved_lkey,
2594 struct ecore_rdma_create_qp_in_params *params)
2600 QL_DPRINT12(ha, "enter\n");
2602 /* QP handle to be written in an async event */
2603 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
2604 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
2606 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
2607 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
2608 params->pd = pd->pd_id;
2609 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
2610 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
2611 params->stats_queue = 0;
2613 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
2616 /* QP is associated with SRQ instead of RQ */
2617 params->srq_id = qp->srq->srq_id;
2618 params->use_srq = true;
2619 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
2620 params->srq_id, params->use_srq);
2625 params->use_srq = false;
2627 QL_DPRINT12(ha, "exit\n");
2632 qlnxr_qp_user_print( struct qlnxr_dev *dev,
2633 struct qlnxr_qp *qp)
2635 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "
2636 "rq_addr=0x%llx, rq_len=%zd\n",
2637 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
2643 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id)
2651 QL_DPRINT12(ha, "enter\n");
2653 if (!QLNX_IS_IWARP(dev))
2657 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {
2658 QL_DPRINT11(ha, "idr_pre_get failed\n");
2662 mtx_lock(&dev->idr_lock);
2664 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);
2666 mtx_unlock(&dev->idr_lock);
2668 } while (rc == -EAGAIN);
2670 QL_DPRINT12(ha, "exit [%d]\n", rc);
2676 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id)
2682 QL_DPRINT12(ha, "enter\n");
2684 if (!QLNX_IS_IWARP(dev))
2687 mtx_lock(&dev->idr_lock);
2688 idr_remove(&dev->qpidr, id);
2689 mtx_unlock(&dev->idr_lock);
2691 QL_DPRINT12(ha, "exit \n");
2697 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev,
2698 struct qlnxr_qp *qp,
2699 struct ecore_rdma_create_qp_out_params *out_params)
2705 QL_DPRINT12(ha, "enter\n");
2707 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
2708 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
2710 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
2714 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);
2718 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
2719 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
2721 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
2724 QL_DPRINT12(ha, "exit\n");
2729 qlnxr_create_user_qp(struct qlnxr_dev *dev,
2730 struct qlnxr_qp *qp,
2732 struct ib_udata *udata,
2733 struct ib_qp_init_attr *attrs)
2735 struct ecore_rdma_destroy_qp_out_params d_out_params;
2736 struct ecore_rdma_create_qp_in_params in_params;
2737 struct ecore_rdma_create_qp_out_params out_params;
2738 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
2739 struct ib_ucontext *ib_ctx = NULL;
2740 struct qlnxr_ucontext *ctx = NULL;
2741 struct qlnxr_create_qp_ureq ureq;
2742 int alloc_and_init = QLNX_IS_ROCE(dev);
2748 QL_DPRINT12(ha, "enter\n");
2750 ib_ctx = ibpd->uobject->context;
2751 ctx = get_qlnxr_ucontext(ib_ctx);
2753 memset(&ureq, 0, sizeof(ureq));
2754 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
2757 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);
2761 /* SQ - read access only (0), dma sync not required (0) */
2762 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
2766 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2771 /* RQ - read access only (0), dma sync not required (0) */
2772 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
2777 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2782 memset(&in_params, 0, sizeof(in_params));
2783 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
2784 in_params.qp_handle_lo = ureq.qp_handle_lo;
2785 in_params.qp_handle_hi = ureq.qp_handle_hi;
2786 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
2787 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
2790 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
2791 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
2794 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);
2796 if (!qp->ecore_qp) {
2798 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
2802 if (QLNX_IS_IWARP(dev))
2803 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params);
2805 qp->qp_id = out_params.qp_id;
2806 qp->icid = out_params.icid;
2808 rc = qlnxr_copy_qp_uresp(dev, qp, udata);
2811 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n");
2815 qlnxr_qp_user_print(dev, qp);
2817 QL_DPRINT12(ha, "exit\n");
2820 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
2823 QL_DPRINT12(ha, "fatal fault\n");
2826 qlnxr_cleanup_user(dev, qp);
2828 QL_DPRINT12(ha, "exit[%d]\n", rc);
2833 qlnxr_set_roce_db_info(struct qlnxr_dev *dev,
2834 struct qlnxr_qp *qp)
2840 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2842 qp->sq.db = dev->db_addr +
2843 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2844 qp->sq.db_data.data.icid = qp->icid + 1;
2847 qp->rq.db = dev->db_addr +
2848 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2849 qp->rq.db_data.data.icid = qp->icid;
2852 QL_DPRINT12(ha, "exit\n");
2857 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev,
2858 struct qlnxr_qp *qp)
2865 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2867 qp->sq.db = dev->db_addr +
2868 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2869 qp->sq.db_data.data.icid = qp->icid;
2872 qp->rq.db = dev->db_addr +
2873 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2874 qp->rq.db_data.data.icid = qp->icid;
2876 qp->rq.iwarp_db2 = dev->db_addr +
2877 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2878 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2879 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2883 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"
2884 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"
2885 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"
2886 " qp->rq.iwarp_db2.data.prod_val =0x%x\n",
2887 qp->sq.db, qp->sq.db_data.data.icid,
2888 qp->rq.db, qp->rq.db_data.data.icid,
2889 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,
2890 qp->rq.iwarp_db2_data.data.value);
2892 QL_DPRINT12(ha, "exit\n");
2897 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev,
2898 struct qlnxr_qp *qp,
2899 struct ecore_rdma_create_qp_in_params *in_params,
2903 struct ecore_rdma_create_qp_out_params out_params;
2909 QL_DPRINT12(ha, "enter\n");
2911 rc = ecore_chain_alloc(
2913 ECORE_CHAIN_USE_TO_PRODUCE,
2914 ECORE_CHAIN_MODE_PBL,
2915 ECORE_CHAIN_CNT_TYPE_U32,
2917 QLNXR_SQE_ELEMENT_SIZE,
2922 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);
2926 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);
2927 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);
2930 rc = ecore_chain_alloc(
2932 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2933 ECORE_CHAIN_MODE_PBL,
2934 ECORE_CHAIN_CNT_TYPE_U32,
2936 QLNXR_RQE_ELEMENT_SIZE,
2942 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);
2946 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);
2947 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);
2950 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
2952 if (!qp->ecore_qp) {
2953 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");
2957 qp->qp_id = out_params.qp_id;
2958 qp->icid = out_params.icid;
2960 qlnxr_set_roce_db_info(dev, qp);
2962 QL_DPRINT12(ha, "exit\n");
2967 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev,
2968 struct qlnxr_qp *qp,
2969 struct ecore_rdma_create_qp_in_params *in_params,
2973 struct ecore_rdma_destroy_qp_out_params d_out_params;
2974 struct ecore_rdma_create_qp_out_params out_params;
2975 struct ecore_chain_ext_pbl ext_pbl;
2981 QL_DPRINT12(ha, "enter\n");
2983 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,
2984 QLNXR_SQE_ELEMENT_SIZE,
2985 ECORE_CHAIN_MODE_PBL);
2986 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,
2987 QLNXR_RQE_ELEMENT_SIZE,
2988 ECORE_CHAIN_MODE_PBL);
2990 QL_DPRINT12(ha, "n_sq_elems = 0x%x"
2991 " n_rq_elems = 0x%x in_params\n"
2992 "\t\t\tqp_handle_lo\t\t= 0x%08x\n"
2993 "\t\t\tqp_handle_hi\t\t= 0x%08x\n"
2994 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"
2995 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"
2996 "\t\t\tuse_srq\t\t\t= 0x%x\n"
2997 "\t\t\tsignal_all\t\t= 0x%x\n"
2998 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"
2999 "\t\t\tpd\t\t\t= 0x%x\n"
3000 "\t\t\tdpi\t\t\t= 0x%x\n"
3001 "\t\t\tsq_cq_id\t\t\t= 0x%x\n"
3002 "\t\t\tsq_num_pages\t\t= 0x%x\n"
3003 "\t\t\tsq_pbl_ptr\t\t= %p\n"
3004 "\t\t\tmax_sq_sges\t\t= 0x%x\n"
3005 "\t\t\trq_cq_id\t\t\t= 0x%x\n"
3006 "\t\t\trq_num_pages\t\t= 0x%x\n"
3007 "\t\t\trq_pbl_ptr\t\t= %p\n"
3008 "\t\t\tsrq_id\t\t\t= 0x%x\n"
3009 "\t\t\tstats_queue\t\t= 0x%x\n",
3010 n_sq_elems, n_rq_elems,
3011 in_params->qp_handle_lo,
3012 in_params->qp_handle_hi,
3013 in_params->qp_handle_async_lo,
3014 in_params->qp_handle_async_hi,
3016 in_params->signal_all,
3017 in_params->fmr_and_reserved_lkey,
3020 in_params->sq_cq_id,
3021 in_params->sq_num_pages,
3022 (void *)in_params->sq_pbl_ptr,
3023 in_params->max_sq_sges,
3024 in_params->rq_cq_id,
3025 in_params->rq_num_pages,
3026 (void *)in_params->rq_pbl_ptr,
3028 in_params->stats_queue );
3030 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params));
3031 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl));
3033 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
3035 if (!qp->ecore_qp) {
3036 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
3040 /* Now we allocate the chain */
3041 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
3042 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
3044 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3045 "ext_pbl.p_pbl_phys = %p\n",
3046 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3048 rc = ecore_chain_alloc(
3050 ECORE_CHAIN_USE_TO_PRODUCE,
3051 ECORE_CHAIN_MODE_PBL,
3052 ECORE_CHAIN_CNT_TYPE_U32,
3054 QLNXR_SQE_ELEMENT_SIZE,
3060 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);
3064 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
3065 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
3067 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3068 "ext_pbl.p_pbl_phys = %p\n",
3069 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3072 rc = ecore_chain_alloc(
3074 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
3075 ECORE_CHAIN_MODE_PBL,
3076 ECORE_CHAIN_CNT_TYPE_U32,
3078 QLNXR_RQE_ELEMENT_SIZE,
3083 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"
3084 " failed rc = %d\n", rc);
3089 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",
3090 out_params.qp_id, out_params.icid);
3092 qp->qp_id = out_params.qp_id;
3093 qp->icid = out_params.icid;
3095 qlnxr_set_iwarp_db_info(dev, qp);
3097 QL_DPRINT12(ha, "exit\n");
3101 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
3103 QL_DPRINT12(ha, "exit rc = %d\n", rc);
3108 qlnxr_create_kernel_qp(struct qlnxr_dev *dev,
3109 struct qlnxr_qp *qp,
3111 struct ib_qp_init_attr *attrs)
3113 struct ecore_rdma_create_qp_in_params in_params;
3114 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3119 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3124 QL_DPRINT12(ha, "enter\n");
3126 memset(&in_params, 0, sizeof(in_params));
3128 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in
3129 * the ring. The ring should allow at least a single WR, even if the
3130 * user requested none, due to allocation issues.
3131 * We should add an extra WR since the prod and cons indices of
3132 * wqe_wr_id are managed in such a way that the WQ is considered full
3133 * when (prod+1)%max_wr==cons. We currently don't do that because we
3134 * double the number of entries due an iSER issue that pushes far more
3135 * WRs than indicated. If we decline its ib_post_send() then we get
3136 * error prints in the dmesg we'd like to avoid.
3138 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
3141 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
3143 if (!qp->wqe_wr_id) {
3144 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n");
3148 /* QP handle to be written in CQE */
3149 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp);
3150 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp);
3152 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in
3153 * the ring. There ring should allow at least a single WR, even if the
3154 * user requested none, due to allocation issues.
3156 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);
3158 /* Allocate driver internal RQ array */
3160 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
3162 if (!qp->rqe_wr_id) {
3163 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n");
3164 kfree(qp->wqe_wr_id);
3169 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
3171 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
3172 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
3174 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
3175 in_params.fmr_and_reserved_lkey = true;
3176 in_params.pd = pd->pd_id;
3177 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
3178 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
3179 in_params.stats_queue = 0;
3181 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
3184 /* QP is associated with SRQ instead of RQ */
3185 in_params.srq_id = qp->srq->srq_id;
3186 in_params.use_srq = true;
3187 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
3188 in_params.srq_id, in_params.use_srq);
3190 in_params.srq_id = 0;
3191 in_params.use_srq = false;
3194 n_sq_entries = attrs->cap.max_send_wr;
3195 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);
3196 n_sq_entries = max_t(u32, n_sq_entries, 1);
3197 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
3199 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
3201 if (QLNX_IS_ROCE(dev)) {
3202 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,
3203 n_sq_elems, n_rq_elems);
3205 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,
3206 n_sq_elems, n_rq_elems);
3210 qlnxr_cleanup_kernel(dev, qp);
3212 QL_DPRINT12(ha, "exit [%d]\n", rc);
3217 qlnxr_create_qp(struct ib_pd *ibpd,
3218 struct ib_qp_init_attr *attrs,
3219 struct ib_udata *udata)
3221 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);
3222 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3223 struct qlnxr_qp *qp;
3229 QL_DPRINT12(ha, "enter\n");
3231 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);
3233 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);
3237 QL_DPRINT12(ha, "called from %s, event_handle=%p,"
3238 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
3239 (udata ? "user library" : "kernel"),
3240 attrs->event_handler, pd,
3241 get_qlnxr_cq(attrs->send_cq),
3242 get_qlnxr_cq(attrs->send_cq)->icid,
3243 get_qlnxr_cq(attrs->recv_cq),
3244 get_qlnxr_cq(attrs->recv_cq)->icid);
3246 qp = qlnx_zalloc(sizeof(struct qlnxr_qp));
3249 QL_DPRINT11(ha, "kzalloc(qp) failed\n");
3250 return ERR_PTR(-ENOMEM);
3253 qlnxr_set_common_qp_params(dev, qp, pd, attrs);
3255 if (attrs->qp_type == IB_QPT_GSI) {
3256 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n");
3257 return qlnxr_create_gsi_qp(dev, attrs, qp);
3261 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);
3264 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n");
3268 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);
3271 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n");
3276 qp->ibqp.qp_num = qp->qp_id;
3278 rc = qlnxr_idr_add(dev, qp, qp->qp_id);
3281 QL_DPRINT11(ha, "qlnxr_idr_add failed\n");
3285 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);
3291 QL_DPRINT12(ha, "failed exit\n");
3292 return ERR_PTR(-EFAULT);
3295 static enum ib_qp_state
3296 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state)
3298 enum ib_qp_state state = IB_QPS_ERR;
3301 case ECORE_ROCE_QP_STATE_RESET:
3302 state = IB_QPS_RESET;
3305 case ECORE_ROCE_QP_STATE_INIT:
3306 state = IB_QPS_INIT;
3309 case ECORE_ROCE_QP_STATE_RTR:
3313 case ECORE_ROCE_QP_STATE_RTS:
3317 case ECORE_ROCE_QP_STATE_SQD:
3321 case ECORE_ROCE_QP_STATE_ERR:
3325 case ECORE_ROCE_QP_STATE_SQE:
3332 static enum ecore_roce_qp_state
3333 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state)
3335 enum ecore_roce_qp_state ecore_qp_state;
3337 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3341 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET;
3345 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT;
3349 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR;
3353 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS;
3357 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD;
3361 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3365 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3369 return (ecore_qp_state);
3373 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph)
3375 ecore_chain_reset(&qph->pbl);
3376 qph->prod = qph->cons = 0;
3378 qph->db_data.data.value = cpu_to_le16(0);
3384 qlnxr_update_qp_state(struct qlnxr_dev *dev,
3385 struct qlnxr_qp *qp,
3386 enum ecore_roce_qp_state new_state)
3390 struct ecore_dev *cdev;
3396 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",
3397 qp, new_state, qp->state);
3399 if (new_state == qp->state) {
3403 switch (qp->state) {
3404 case ECORE_ROCE_QP_STATE_RESET:
3405 switch (new_state) {
3406 case ECORE_ROCE_QP_STATE_INIT:
3407 qp->prev_wqe_size = 0;
3408 qlnxr_reset_qp_hwq_info(&qp->sq);
3410 qlnxr_reset_qp_hwq_info(&qp->rq);
3417 case ECORE_ROCE_QP_STATE_INIT:
3419 switch (new_state) {
3420 case ECORE_ROCE_QP_STATE_RTR:
3421 /* Update doorbell (in case post_recv was done before move to RTR) */
3425 //writel(qp->rq.db_data.raw, qp->rq.db);
3426 //if (QLNX_IS_IWARP(dev))
3427 // writel(qp->rq.iwarp_db2_data.raw,
3428 // qp->rq.iwarp_db2);
3430 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
3431 (uint8_t *)cdev->doorbells);
3433 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
3434 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
3436 if (QLNX_IS_IWARP(dev)) {
3437 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
3438 (uint8_t *)cdev->doorbells);
3439 bus_write_4(ha->pci_dbells, reg_addr,\
3440 qp->rq.iwarp_db2_data.raw);
3441 bus_barrier(ha->pci_dbells, 0, 0,\
3442 BUS_SPACE_BARRIER_READ);
3448 case ECORE_ROCE_QP_STATE_ERR:
3449 /* TBD:flush qps... */
3452 /* invalid state change. */
3457 case ECORE_ROCE_QP_STATE_RTR:
3459 switch (new_state) {
3460 case ECORE_ROCE_QP_STATE_RTS:
3462 case ECORE_ROCE_QP_STATE_ERR:
3465 /* invalid state change. */
3470 case ECORE_ROCE_QP_STATE_RTS:
3472 switch (new_state) {
3473 case ECORE_ROCE_QP_STATE_SQD:
3475 case ECORE_ROCE_QP_STATE_ERR:
3478 /* invalid state change. */
3483 case ECORE_ROCE_QP_STATE_SQD:
3485 switch (new_state) {
3486 case ECORE_ROCE_QP_STATE_RTS:
3487 case ECORE_ROCE_QP_STATE_ERR:
3490 /* invalid state change. */
3495 case ECORE_ROCE_QP_STATE_ERR:
3497 switch (new_state) {
3498 case ECORE_ROCE_QP_STATE_RESET:
3499 if ((qp->rq.prod != qp->rq.cons) ||
3500 (qp->sq.prod != qp->sq.cons)) {
3502 "Error->Reset with rq/sq "
3503 "not empty rq.prod=0x%x rq.cons=0x%x"
3504 " sq.prod=0x%x sq.cons=0x%x\n",
3505 qp->rq.prod, qp->rq.cons,
3506 qp->sq.prod, qp->sq.cons);
3520 QL_DPRINT12(ha, "exit\n");
3525 qlnxr_modify_qp(struct ib_qp *ibqp,
3526 struct ib_qp_attr *attr,
3528 struct ib_udata *udata)
3531 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3532 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);
3533 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
3534 enum ib_qp_state old_qp_state, new_qp_state;
3535 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3541 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",
3542 qp, attr_mask, attr->qp_state, udata);
3544 old_qp_state = qlnxr_get_ibqp_state(qp->state);
3545 if (attr_mask & IB_QP_STATE)
3546 new_qp_state = attr->qp_state;
3548 new_qp_state = old_qp_state;
3550 if (QLNX_IS_ROCE(dev)) {
3551 #if __FreeBSD_version >= 1100000
3552 if (!ib_modify_qp_is_ok(old_qp_state,
3556 IB_LINK_LAYER_ETHERNET)) {
3558 "invalid attribute mask=0x%x"
3559 " specified for qpn=0x%x of type=0x%x \n"
3560 " old_qp_state=0x%x, new_qp_state=0x%x\n",
3561 attr_mask, qp->qp_id, ibqp->qp_type,
3562 old_qp_state, new_qp_state);
3567 if (!ib_modify_qp_is_ok(old_qp_state,
3572 "invalid attribute mask=0x%x"
3573 " specified for qpn=0x%x of type=0x%x \n"
3574 " old_qp_state=0x%x, new_qp_state=0x%x\n",
3575 attr_mask, qp->qp_id, ibqp->qp_type,
3576 old_qp_state, new_qp_state);
3581 #endif /* #if __FreeBSD_version >= 1100000 */
3583 /* translate the masks... */
3584 if (attr_mask & IB_QP_STATE) {
3585 SET_FIELD(qp_params.modify_flags,
3586 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
3587 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);
3590 // TBD consider changing ecore to be a flag as well...
3591 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
3592 qp_params.sqd_async = true;
3594 if (attr_mask & IB_QP_PKEY_INDEX) {
3595 SET_FIELD(qp_params.modify_flags,
3596 ECORE_ROCE_MODIFY_QP_VALID_PKEY,
3598 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {
3603 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT;
3606 if (attr_mask & IB_QP_QKEY) {
3607 qp->qkey = attr->qkey;
3610 /* tbd consider splitting in ecore.. */
3611 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3612 SET_FIELD(qp_params.modify_flags,
3613 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
3614 qp_params.incoming_rdma_read_en =
3615 attr->qp_access_flags & IB_ACCESS_REMOTE_READ;
3616 qp_params.incoming_rdma_write_en =
3617 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;
3618 qp_params.incoming_atomic_en =
3619 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;
3622 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
3623 if (attr_mask & IB_QP_PATH_MTU) {
3624 if (attr->path_mtu < IB_MTU_256 ||
3625 attr->path_mtu > IB_MTU_4096) {
3627 "Only MTU sizes of 256, 512, 1024,"
3628 " 2048 and 4096 are supported "
3629 " attr->path_mtu = [%d]\n",
3635 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
3637 iboe_get_mtu(dev->ha->ifp->if_mtu)));
3641 qp->mtu = ib_mtu_enum_to_int(
3642 iboe_get_mtu(dev->ha->ifp->if_mtu));
3643 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",
3647 SET_FIELD(qp_params.modify_flags,
3648 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR,
3651 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
3652 qp_params.flow_label = attr->ah_attr.grh.flow_label;
3653 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
3655 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
3657 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params);
3659 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);
3663 qp_params.use_local_mac = true;
3664 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
3666 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",
3667 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
3668 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
3669 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",
3670 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
3671 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
3673 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",
3674 qp_params.remote_mac_addr[0],
3675 qp_params.remote_mac_addr[1],
3676 qp_params.remote_mac_addr[2],
3677 qp_params.remote_mac_addr[3],
3678 qp_params.remote_mac_addr[4],
3679 qp_params.remote_mac_addr[5]);
3681 qp_params.mtu = qp->mtu;
3684 if (qp_params.mtu == 0) {
3685 /* stay with current MTU */
3687 qp_params.mtu = qp->mtu;
3689 qp_params.mtu = ib_mtu_enum_to_int(
3690 iboe_get_mtu(dev->ha->ifp->if_mtu));
3694 if (attr_mask & IB_QP_TIMEOUT) {
3695 SET_FIELD(qp_params.modify_flags, \
3696 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
3698 qp_params.ack_timeout = attr->timeout;
3699 if (attr->timeout) {
3702 /* 12.7.34 LOCAL ACK TIMEOUT
3703 * Value representing the transport (ACK) timeout for
3704 * use by the remote, expressed as (4.096 μS*2Local ACK
3707 /* We use 1UL since the temporal value may be overflow
3710 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
3711 qp_params.ack_timeout = temp; /* FW requires [msec] */
3714 qp_params.ack_timeout = 0; /* infinite */
3716 if (attr_mask & IB_QP_RETRY_CNT) {
3717 SET_FIELD(qp_params.modify_flags,\
3718 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
3719 qp_params.retry_cnt = attr->retry_cnt;
3722 if (attr_mask & IB_QP_RNR_RETRY) {
3723 SET_FIELD(qp_params.modify_flags,
3724 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT,
3726 qp_params.rnr_retry_cnt = attr->rnr_retry;
3729 if (attr_mask & IB_QP_RQ_PSN) {
3730 SET_FIELD(qp_params.modify_flags,
3731 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN,
3733 qp_params.rq_psn = attr->rq_psn;
3734 qp->rq_psn = attr->rq_psn;
3737 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3738 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {
3741 "unsupported max_rd_atomic=%d, supported=%d\n",
3742 attr->max_rd_atomic,
3743 qattr->max_qp_req_rd_atomic_resc);
3747 SET_FIELD(qp_params.modify_flags,
3748 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ,
3750 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
3753 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
3754 SET_FIELD(qp_params.modify_flags,
3755 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER,
3757 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
3760 if (attr_mask & IB_QP_SQ_PSN) {
3761 SET_FIELD(qp_params.modify_flags,
3762 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN,
3764 qp_params.sq_psn = attr->sq_psn;
3765 qp->sq_psn = attr->sq_psn;
3768 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3769 if (attr->max_dest_rd_atomic >
3770 qattr->max_qp_resp_rd_atomic_resc) {
3772 "unsupported max_dest_rd_atomic=%d, "
3774 attr->max_dest_rd_atomic,
3775 qattr->max_qp_resp_rd_atomic_resc);
3781 SET_FIELD(qp_params.modify_flags,
3782 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP,
3784 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
3787 if (attr_mask & IB_QP_DEST_QPN) {
3788 SET_FIELD(qp_params.modify_flags,
3789 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP,
3792 qp_params.dest_qp = attr->dest_qp_num;
3793 qp->dest_qp_num = attr->dest_qp_num;
3797 * Update the QP state before the actual ramrod to prevent a race with
3798 * fast path. Modifying the QP state to error will cause the device to
3799 * flush the CQEs and while polling the flushed CQEs will considered as
3800 * a potential issue if the QP isn't in error state.
3802 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&
3803 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR))
3804 qp->state = ECORE_ROCE_QP_STATE_ERR;
3806 if (qp->qp_type != IB_QPT_GSI)
3807 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
3809 if (attr_mask & IB_QP_STATE) {
3810 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
3811 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);
3812 qp->state = qp_params.new_state;
3816 QL_DPRINT12(ha, "exit\n");
3821 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params)
3823 int ib_qp_acc_flags = 0;
3825 if (params->incoming_rdma_write_en)
3826 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
3827 if (params->incoming_rdma_read_en)
3828 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
3829 if (params->incoming_atomic_en)
3830 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
3831 if (true) /* FIXME -> local write ?? */
3832 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
3834 return ib_qp_acc_flags;
3838 qlnxr_mtu_int_to_enum(u16 mtu)
3840 enum ib_mtu ib_mtu_size;
3844 ib_mtu_size = IB_MTU_256;
3848 ib_mtu_size = IB_MTU_512;
3852 ib_mtu_size = IB_MTU_1024;
3856 ib_mtu_size = IB_MTU_2048;
3860 ib_mtu_size = IB_MTU_4096;
3864 ib_mtu_size = IB_MTU_1024;
3867 return (ib_mtu_size);
3871 qlnxr_query_qp(struct ib_qp *ibqp,
3872 struct ib_qp_attr *qp_attr,
3874 struct ib_qp_init_attr *qp_init_attr)
3877 struct ecore_rdma_query_qp_out_params params;
3878 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3879 struct qlnxr_dev *dev = qp->dev;
3884 QL_DPRINT12(ha, "enter\n");
3886 memset(¶ms, 0, sizeof(params));
3888 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms);
3892 memset(qp_attr, 0, sizeof(*qp_attr));
3893 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3895 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);
3896 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);
3898 /* In some cases in iWARP qelr will ask for the state only */
3899 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) {
3900 QL_DPRINT11(ha, "only state requested\n");
3904 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);
3905 qp_attr->path_mig_state = IB_MIG_MIGRATED;
3906 qp_attr->rq_psn = params.rq_psn;
3907 qp_attr->sq_psn = params.sq_psn;
3908 qp_attr->dest_qp_num = params.dest_qp;
3910 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms);
3912 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "
3913 "path_mtu = %d qp_access_flags = 0x%x\n",
3914 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,
3915 qp_attr->qp_access_flags);
3917 qp_attr->cap.max_send_wr = qp->sq.max_wr;
3918 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
3919 qp_attr->cap.max_send_sge = qp->sq.max_sges;
3920 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
3921 qp_attr->cap.max_inline_data = qp->max_inline_data;
3922 qp_init_attr->cap = qp_attr->cap;
3924 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0],
3925 sizeof(qp_attr->ah_attr.grh.dgid.raw));
3927 qp_attr->ah_attr.grh.flow_label = params.flow_label;
3928 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
3929 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
3930 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
3932 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
3933 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */
3934 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */
3935 qp_attr->timeout = params.timeout;
3936 qp_attr->rnr_retry = params.rnr_retry;
3937 qp_attr->retry_cnt = params.retry_cnt;
3938 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
3939 qp_attr->pkey_index = params.pkey_index;
3940 qp_attr->port_num = 1; /* FIXME -> check this */
3941 qp_attr->ah_attr.src_path_bits = 0;
3942 qp_attr->ah_attr.static_rate = 0;
3943 qp_attr->alt_pkey_index = 0;
3944 qp_attr->alt_port_num = 0;
3945 qp_attr->alt_timeout = 0;
3946 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
3948 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;
3949 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
3950 qp_attr->max_rd_atomic = params.max_rd_atomic;
3951 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;
3953 QL_DPRINT12(ha, "max_inline_data=%d\n",
3954 qp_attr->cap.max_inline_data);
3957 QL_DPRINT12(ha, "exit\n");
3962 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3968 QL_DPRINT12(ha, "enter\n");
3971 ib_umem_release(qp->usq.umem);
3973 qp->usq.umem = NULL;
3976 ib_umem_release(qp->urq.umem);
3978 qp->urq.umem = NULL;
3980 QL_DPRINT12(ha, "exit\n");
3985 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3991 QL_DPRINT12(ha, "enter\n");
3993 if (qlnxr_qp_has_sq(qp)) {
3994 QL_DPRINT12(ha, "freeing SQ\n");
3995 ha->qlnxr_debug = 1;
3996 // ecore_chain_free(dev->cdev, &qp->sq.pbl);
3997 ha->qlnxr_debug = 0;
3998 kfree(qp->wqe_wr_id);
4001 if (qlnxr_qp_has_rq(qp)) {
4002 QL_DPRINT12(ha, "freeing RQ\n");
4003 ha->qlnxr_debug = 1;
4004 // ecore_chain_free(dev->cdev, &qp->rq.pbl);
4005 ha->qlnxr_debug = 0;
4006 kfree(qp->rqe_wr_id);
4009 QL_DPRINT12(ha, "exit\n");
4014 qlnxr_free_qp_resources(struct qlnxr_dev *dev,
4015 struct qlnxr_qp *qp)
4019 struct ecore_rdma_destroy_qp_out_params d_out_params;
4023 QL_DPRINT12(ha, "enter\n");
4026 if (qp->qp_type != IB_QPT_GSI) {
4027 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4033 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4034 qlnxr_cleanup_user(dev, qp);
4036 qlnxr_cleanup_kernel(dev, qp);
4039 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4040 qlnxr_cleanup_user(dev, qp);
4042 qlnxr_cleanup_kernel(dev, qp);
4044 if (qp->qp_type != IB_QPT_GSI) {
4045 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4051 QL_DPRINT12(ha, "exit\n");
4056 qlnxr_destroy_qp(struct ib_qp *ibqp)
4058 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4059 struct qlnxr_dev *dev = qp->dev;
4061 struct ib_qp_attr attr;
4067 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);
4071 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |
4072 ECORE_ROCE_QP_STATE_ERR |
4073 ECORE_ROCE_QP_STATE_INIT))) {
4074 attr.qp_state = IB_QPS_ERR;
4075 attr_mask |= IB_QP_STATE;
4077 /* change the QP state to ERROR */
4078 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL);
4081 if (qp->qp_type == IB_QPT_GSI)
4082 qlnxr_destroy_gsi_qp(dev);
4086 qlnxr_free_qp_resources(dev, qp);
4088 if (atomic_dec_and_test(&qp->refcnt)) {
4089 /* TODO: only for iWARP? */
4090 qlnxr_idr_remove(dev, qp->qp_id);
4094 QL_DPRINT12(ha, "exit\n");
4099 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq)
4101 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
4105 sge_data_len(struct ib_sge *sg_list, int num_sge)
4108 for (i = 0; i < num_sge; i++)
4109 len += sg_list[i].length;
4114 swap_wqe_data64(u64 *p)
4118 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
4119 *p = cpu_to_be64(cpu_to_le64(*p));
4123 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev,
4124 struct qlnxr_qp *qp,
4126 struct ib_send_wr *wr,
4127 struct ib_send_wr **bad_wr,
4132 char *seg_prt, *wqe;
4133 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
4138 QL_DPRINT12(ha, "enter[%d]\n", data_size);
4140 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
4142 "Too much inline data in WR:[%d, %d]\n",
4143 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE);
4154 seg_prt = wqe = NULL;
4157 /* copy data inline */
4158 for (i = 0; i < wr->num_sge; i++) {
4159 u32 len = wr->sg_list[i].length;
4160 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
4165 /* new segment required */
4167 wqe = (char *)ecore_chain_produce(&qp->sq.pbl);
4169 seg_siz = sizeof(struct rdma_sq_common_wqe);
4173 /* calculate currently allowed length */
4174 cur = MIN(len, seg_siz);
4176 memcpy(seg_prt, src, cur);
4178 /* update segment variables */
4181 /* update sge variables */
4185 /* swap fully-completed segments */
4187 swap_wqe_data64((u64 *)wqe);
4191 /* swap last not completed segment */
4193 swap_wqe_data64((u64 *)wqe);
4195 QL_DPRINT12(ha, "exit\n");
4200 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp,
4201 u8 *wqe_size, struct ib_send_wr *wr)
4209 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);
4211 for (i = 0; i < wr->num_sge; i++) {
4212 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);
4214 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
4215 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
4216 sge->length = cpu_to_le32(wr->sg_list[i].length);
4217 data_size += wr->sg_list[i].length;
4221 *wqe_size += wr->num_sge;
4223 QL_DPRINT12(ha, "exit data_size = %d\n", data_size);
4228 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev,
4229 struct qlnxr_qp *qp,
4230 struct rdma_sq_rdma_wqe_1st *rwqe,
4231 struct rdma_sq_rdma_wqe_2nd *rwqe2,
4232 struct ib_send_wr *wr,
4233 struct ib_send_wr **bad_wr)
4240 QL_DPRINT12(ha, "enter\n");
4242 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
4243 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);
4245 if (wr->send_flags & IB_SEND_INLINE) {
4247 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
4248 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,
4249 wr, bad_wr, &rwqe->flags, flags);
4252 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);
4254 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4260 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev,
4261 struct qlnxr_qp *qp,
4262 struct rdma_sq_send_wqe *swqe,
4263 struct rdma_sq_send_wqe *swqe2,
4264 struct ib_send_wr *wr,
4265 struct ib_send_wr **bad_wr)
4272 QL_DPRINT12(ha, "enter\n");
4274 memset(swqe2, 0, sizeof(*swqe2));
4276 if (wr->send_flags & IB_SEND_INLINE) {
4278 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
4279 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,
4280 wr, bad_wr, &swqe->flags, flags);
4283 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);
4285 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4291 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info)
4297 int work = info->completed - info->completed_handled - 1;
4299 QL_DPRINT12(ha, "enter [%d]\n", work);
4301 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
4302 struct qlnxr_pbl *pbl;
4304 /* Free all the page list that are possible to be freed
4305 * (all the ones that were invalidated), under the assumption
4306 * that if an FMR was completed successfully that means that
4307 * if there was an invalidate operation before it also ended
4309 pbl = list_first_entry(&info->inuse_pbl_list,
4312 list_del(&pbl->list_entry);
4313 list_add_tail(&pbl->list_entry, &info->free_pbl_list);
4314 info->completed_handled++;
4317 QL_DPRINT12(ha, "exit\n");
4321 #if __FreeBSD_version >= 1102000
4323 static int qlnxr_prepare_reg(struct qlnxr_qp *qp,
4324 struct rdma_sq_fmr_wqe_1st *fwqe1,
4325 struct ib_reg_wr *wr)
4327 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);
4328 struct rdma_sq_fmr_wqe_2nd *fwqe2;
4330 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);
4331 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
4332 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
4333 fwqe1->l_key = wr->key;
4335 fwqe2->access_ctrl = 0;
4337 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
4338 !!(wr->access & IB_ACCESS_REMOTE_READ));
4339 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
4340 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
4341 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
4342 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
4343 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
4344 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
4345 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
4346 fwqe2->fmr_ctrl = 0;
4348 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4349 ilog2(mr->ibmr.page_size) - 12);
4351 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */
4352 fwqe2->length_lo = mr->ibmr.length;
4353 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
4354 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
4356 qp->wqe_wr_id[qp->sq.prod].mr = mr;
4364 build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr,
4365 struct mr_info *info)
4369 int num_pbes, total_num_pbes = 0;
4370 struct regpair *pbe;
4371 struct qlnxr_pbl *pbl_tbl = info->pbl_table;
4372 struct qlnxr_pbl_info *pbl_info = &info->pbl_info;
4377 QL_DPRINT12(ha, "enter\n");
4379 pbe = (struct regpair *)pbl_tbl->va;
4382 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
4383 buf_addr = wr->wr.fast_reg.page_list->page_list[i];
4384 pbe->lo = cpu_to_le32((u32)buf_addr);
4385 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr));
4391 if (total_num_pbes == pbl_info->num_pbes)
4394 /* if the given pbl is full storing the pbes,
4398 (pbl_info->pbl_size / sizeof(u64))) {
4400 pbe = (struct regpair *)pbl_tbl->va;
4404 QL_DPRINT12(ha, "exit\n");
4410 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info)
4417 QL_DPRINT12(ha, "enter\n");
4419 if (info->completed == 0) {
4420 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n");
4425 qlnx_handle_completed_mrs(dev, info);
4427 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list);
4429 if (list_empty(&info->free_pbl_list)) {
4430 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info,
4433 info->pbl_table = list_first_entry(&info->free_pbl_list,
4436 list_del(&info->pbl_table->list_entry);
4439 if (!info->pbl_table)
4442 QL_DPRINT12(ha, "exit\n");
4447 qlnxr_prepare_fmr(struct qlnxr_qp *qp,
4448 struct rdma_sq_fmr_wqe_1st *fwqe1,
4449 struct ib_send_wr *wr)
4451 struct qlnxr_dev *dev = qp->dev;
4453 struct qlnxr_fast_reg_page_list *frmr_list =
4454 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list);
4455 struct rdma_sq_fmr_wqe *fwqe2 =
4456 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl);
4462 QL_DPRINT12(ha, "enter\n");
4464 if (wr->wr.fast_reg.page_list_len == 0)
4467 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info);
4471 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start);
4472 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start);
4473 fwqe1->l_key = wr->wr.fast_reg.rkey;
4475 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ,
4476 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ));
4477 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE,
4478 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE));
4479 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC,
4480 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC));
4481 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1);
4482 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE,
4483 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE));
4485 fwqe2->fmr_ctrl = 0;
4487 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4488 ilog2(1 << wr->wr.fast_reg.page_shift) - 12);
4489 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0);
4491 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */
4492 fwqe2->length_lo = wr->wr.fast_reg.length;
4493 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa);
4494 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa);
4496 /* produce another wqe for fwqe3 */
4497 ecore_chain_produce(&qp->sq.pbl);
4499 fbo = wr->wr.fast_reg.iova_start -
4500 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
4502 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x"
4503 " length = %x pbl_addr %x:%x\n",
4504 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey,
4505 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo,
4506 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo);
4508 build_frmr_pbes(dev, wr, &frmr_list->info);
4510 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list;
4512 QL_DPRINT12(ha, "exit\n");
4516 #endif /* #if __FreeBSD_version >= 1102000 */
4518 static enum ib_wc_opcode
4519 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
4522 case IB_WR_RDMA_WRITE:
4523 case IB_WR_RDMA_WRITE_WITH_IMM:
4524 return IB_WC_RDMA_WRITE;
4525 case IB_WR_SEND_WITH_IMM:
4527 case IB_WR_SEND_WITH_INV:
4529 case IB_WR_RDMA_READ:
4530 return IB_WC_RDMA_READ;
4531 case IB_WR_ATOMIC_CMP_AND_SWP:
4532 return IB_WC_COMP_SWAP;
4533 case IB_WR_ATOMIC_FETCH_AND_ADD:
4534 return IB_WC_FETCH_ADD;
4536 #if __FreeBSD_version >= 1102000
4538 return IB_WC_REG_MR;
4540 case IB_WR_FAST_REG_MR:
4541 return IB_WC_FAST_REG_MR;
4542 #endif /* #if __FreeBSD_version >= 1102000 */
4544 case IB_WR_LOCAL_INV:
4545 return IB_WC_LOCAL_INV;
4551 qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr)
4553 int wq_is_full, err_wr, pbl_is_full;
4554 struct qlnxr_dev *dev = qp->dev;
4559 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr);
4561 /* prevent SQ overflow and/or processing of a bad WR */
4562 err_wr = wr->num_sge > qp->sq.max_sges;
4563 wq_is_full = qlnxr_wq_is_full(&qp->sq);
4564 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <
4565 QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
4566 if (wq_is_full || err_wr || pbl_is_full) {
4568 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {
4569 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;
4572 "error: WQ is full. Post send on QP failed"
4573 " (this error appears only once) "
4574 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4575 qp, wr, qp->err_bitmap);
4579 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {
4580 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;
4583 "error: WQ is bad. Post send on QP failed"
4584 " (this error appears only once) "
4585 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4586 qp, wr, qp->err_bitmap);
4590 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {
4591 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;
4594 "error: WQ PBL is full. Post send on QP failed"
4595 " (this error appears only once) "
4596 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4597 qp, wr, qp->err_bitmap);
4601 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr);
4606 qlnxr_post_send(struct ib_qp *ibqp,
4607 struct ib_send_wr *wr,
4608 struct ib_send_wr **bad_wr)
4610 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device);
4611 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4612 unsigned long flags;
4613 int status = 0, rc = 0;
4621 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4624 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4627 if (qp->qp_type == IB_QPT_GSI)
4628 return qlnxr_gsi_post_send(ibqp, wr, bad_wr);
4630 spin_lock_irqsave(&qp->q_lock, flags);
4632 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&
4633 (qp->state != ECORE_ROCE_QP_STATE_ERR) &&
4634 (qp->state != ECORE_ROCE_QP_STATE_SQD)) {
4635 spin_unlock_irqrestore(&qp->q_lock, flags);
4637 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",
4638 qp->icid, qp->state);
4643 QL_DPRINT11(ha, "Got an empty post send???\n");
4647 struct rdma_sq_common_wqe *wqe;
4648 struct rdma_sq_send_wqe *swqe;
4649 struct rdma_sq_send_wqe *swqe2;
4650 struct rdma_sq_rdma_wqe_1st *rwqe;
4651 struct rdma_sq_rdma_wqe_2nd *rwqe2;
4652 struct rdma_sq_local_inv_wqe *iwqe;
4653 struct rdma_sq_atomic_wqe *awqe1;
4654 struct rdma_sq_atomic_wqe *awqe2;
4655 struct rdma_sq_atomic_wqe *awqe3;
4656 struct rdma_sq_fmr_wqe_1st *fwqe1;
4658 if (!qlnxr_can_post_send(qp, wr)) {
4664 wqe = ecore_chain_produce(&qp->sq.pbl);
4666 qp->wqe_wr_id[qp->sq.prod].signaled =
4667 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
4671 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<
4672 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT);
4674 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \
4675 !!(wr->send_flags & IB_SEND_SOLICITED));
4677 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||
4680 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
4681 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \
4682 !!(wr->send_flags & IB_SEND_FENCE));
4684 wqe->prev_wqe_size = qp->prev_wqe_size;
4686 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);
4688 switch (wr->opcode) {
4689 case IB_WR_SEND_WITH_IMM:
4691 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
4692 swqe = (struct rdma_sq_send_wqe *)wqe;
4694 swqe2 = (struct rdma_sq_send_wqe *)
4695 ecore_chain_produce(&qp->sq.pbl);
4696 swqe->inv_key_or_imm_data =
4697 cpu_to_le32(wr->ex.imm_data);
4698 swqe->length = cpu_to_le32(
4699 qlnxr_prepare_sq_send_data(dev,
4700 qp, swqe, swqe2, wr,
4703 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4704 qp->prev_wqe_size = swqe->wqe_size;
4705 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4707 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",
4708 swqe->length, wr->ex.imm_data);
4714 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
4715 swqe = (struct rdma_sq_send_wqe *)wqe;
4718 swqe2 = (struct rdma_sq_send_wqe *)
4719 ecore_chain_produce(&qp->sq.pbl);
4720 swqe->length = cpu_to_le32(
4721 qlnxr_prepare_sq_send_data(dev,
4722 qp, swqe, swqe2, wr,
4724 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4725 qp->prev_wqe_size = swqe->wqe_size;
4726 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4728 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n",
4733 case IB_WR_SEND_WITH_INV:
4735 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
4736 swqe = (struct rdma_sq_send_wqe *)wqe;
4737 swqe2 = (struct rdma_sq_send_wqe *)
4738 ecore_chain_produce(&qp->sq.pbl);
4740 swqe->inv_key_or_imm_data =
4741 cpu_to_le32(wr->ex.invalidate_rkey);
4742 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,
4743 qp, swqe, swqe2, wr, bad_wr));
4744 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4745 qp->prev_wqe_size = swqe->wqe_size;
4746 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4748 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n",
4752 case IB_WR_RDMA_WRITE_WITH_IMM:
4754 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
4755 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4758 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
4759 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4760 ecore_chain_produce(&qp->sq.pbl);
4761 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4762 qp, rwqe, rwqe2, wr, bad_wr));
4763 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4764 qp->prev_wqe_size = rwqe->wqe_size;
4765 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4768 "RDMA WRITE w/ IMM length = %d imm data=%x\n",
4769 rwqe->length, rwqe->imm_data);
4773 case IB_WR_RDMA_WRITE:
4775 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
4776 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4779 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4780 ecore_chain_produce(&qp->sq.pbl);
4781 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4782 qp, rwqe, rwqe2, wr, bad_wr));
4783 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4784 qp->prev_wqe_size = rwqe->wqe_size;
4785 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4788 "RDMA WRITE w/o IMM length = %d\n",
4793 case IB_WR_RDMA_READ_WITH_INV:
4796 "RDMA READ WITH INVALIDATE not supported\n");
4803 case IB_WR_RDMA_READ:
4805 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
4806 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4809 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4810 ecore_chain_produce(&qp->sq.pbl);
4811 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4812 qp, rwqe, rwqe2, wr, bad_wr));
4814 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4815 qp->prev_wqe_size = rwqe->wqe_size;
4816 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4818 QL_DPRINT12(ha, "RDMA READ length = %d\n",
4823 case IB_WR_ATOMIC_CMP_AND_SWP:
4824 case IB_WR_ATOMIC_FETCH_AND_ADD:
4827 "ATOMIC operation = %s\n",
4828 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?
4829 "IB_WR_ATOMIC_CMP_AND_SWP" :
4830 "IB_WR_ATOMIC_FETCH_AND_ADD"));
4832 awqe1 = (struct rdma_sq_atomic_wqe *)wqe;
4833 awqe1->prev_wqe_size = 4;
4835 awqe2 = (struct rdma_sq_atomic_wqe *)
4836 ecore_chain_produce(&qp->sq.pbl);
4838 TYPEPTR_ADDR_SET(awqe2, remote_va, \
4839 atomic_wr(wr)->remote_addr);
4841 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
4843 awqe3 = (struct rdma_sq_atomic_wqe *)
4844 ecore_chain_produce(&qp->sq.pbl);
4846 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
4847 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
4848 TYPEPTR_ADDR_SET(awqe3, swap_data,
4849 atomic_wr(wr)->compare_add);
4851 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
4852 TYPEPTR_ADDR_SET(awqe3, swap_data,
4853 atomic_wr(wr)->swap);
4854 TYPEPTR_ADDR_SET(awqe3, cmp_data,
4855 atomic_wr(wr)->compare_add);
4858 qlnxr_prepare_sq_sges(dev, qp, NULL, wr);
4860 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;
4861 qp->prev_wqe_size = awqe1->prev_wqe_size;
4865 case IB_WR_LOCAL_INV:
4868 "INVALIDATE length (IB_WR_LOCAL_INV)\n");
4870 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
4871 iwqe->prev_wqe_size = 1;
4873 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
4874 iwqe->inv_l_key = wr->ex.invalidate_rkey;
4875 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;
4876 qp->prev_wqe_size = iwqe->prev_wqe_size;
4880 #if __FreeBSD_version >= 1102000
4884 QL_DPRINT12(ha, "IB_WR_REG_MR\n");
4886 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4887 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4888 fwqe1->wqe_size = 2;
4890 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));
4892 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);
4897 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
4898 qp->prev_wqe_size = fwqe1->wqe_size;
4902 case IB_WR_FAST_REG_MR:
4904 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n");
4906 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4907 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4908 fwqe1->prev_wqe_size = 3;
4910 rc = qlnxr_prepare_fmr(qp, fwqe1, wr);
4914 "FAST_MR (IB_WR_FAST_REG_MR) failed"
4920 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size;
4921 qp->prev_wqe_size = fwqe1->prev_wqe_size;
4924 #endif /* #if __FreeBSD_version >= 1102000 */
4928 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);
4937 * restore prod to its position before this WR was processed
4939 ecore_chain_set_prod(&qp->sq.pbl,
4940 le16_to_cpu(qp->sq.db_data.data.value),
4942 /* restore prev_wqe_size */
4943 qp->prev_wqe_size = wqe->prev_wqe_size;
4946 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr);
4947 break; /* out of the loop */
4950 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
4952 qlnxr_inc_sw_prod(&qp->sq);
4954 qp->sq.db_data.data.value++;
4960 * If there was a failure in the first WR then it will be triggered in
4961 * vane. However this is not harmful (as long as the producer value is
4962 * unchanged). For performance reasons we avoid checking for this
4963 * redundant doorbell.
4966 //writel(qp->sq.db_data.raw, qp->sq.db);
4968 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);
4969 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);
4970 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
4974 spin_unlock_irqrestore(&qp->q_lock, flags);
4976 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4983 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq)
4987 /* Calculate number of elements used based on producer
4988 * count and consumer count and subtract it from max
4989 * work request supported so that we get elements left.
4991 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
4993 return hw_srq->max_wr - used;
4997 qlnxr_post_recv(struct ib_qp *ibqp,
4998 struct ib_recv_wr *wr,
4999 struct ib_recv_wr **bad_wr)
5001 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
5002 struct qlnxr_dev *dev = qp->dev;
5003 unsigned long flags;
5010 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5013 QL_DPRINT12(ha, "enter\n");
5015 if (qp->qp_type == IB_QPT_GSI) {
5016 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");
5017 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr);
5021 QL_DPRINT11(ha, "qp->srq [%p]"
5022 " QP is associated with SRQ, cannot post RQ buffers\n",
5027 spin_lock_irqsave(&qp->q_lock, flags);
5029 if (qp->state == ECORE_ROCE_QP_STATE_RESET) {
5030 spin_unlock_irqrestore(&qp->q_lock, flags);
5033 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");
5041 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <
5042 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) ||
5043 (wr->num_sge > qp->rq.max_sges)) {
5048 for (i = 0; i < wr->num_sge; i++) {
5050 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5052 /* first one must include the number of SGE in the list */
5054 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);
5056 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);
5058 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \
5059 wr->sg_list[i].length, flags);
5061 /* Special case of no sges. FW requires between 1-4 sges...
5062 * in this case we need to post 1 sge with length zero. this is
5063 * because rdma write with immediate consumes an RQ. */
5066 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5068 /* first one must include the number of SGE in the list */
5069 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
5070 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
5072 //RQ_SGE_SET(rqe, 0, 0, flags);
5077 rqe->flags = cpu_to_le32(flags);
5082 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
5083 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
5085 qlnxr_inc_sw_prod(&qp->rq);
5089 qp->rq.db_data.data.value++;
5091 // writel(qp->rq.db_data.raw, qp->rq.db);
5093 // if (QLNX_IS_IWARP(dev)) {
5094 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
5095 // mmiowb(); /* for second doorbell */
5098 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
5099 (uint8_t *)ha->cdev.doorbells);
5101 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
5102 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5104 if (QLNX_IS_IWARP(dev)) {
5105 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
5106 (uint8_t *)ha->cdev.doorbells);
5107 bus_write_4(ha->pci_dbells, reg_addr, \
5108 qp->rq.iwarp_db2_data.raw);
5109 bus_barrier(ha->pci_dbells, 0, 0, \
5110 BUS_SPACE_BARRIER_READ);
5116 spin_unlock_irqrestore(&qp->q_lock, flags);
5118 QL_DPRINT12(ha, "exit status = 0x%x\n", status);
5123 /* In fmr we need to increase the number of fmr completed counter for the fmr
5124 * algorithm determining whether we can free a pbl or not.
5125 * we need to perform this whether the work request was signaled or not. for
5126 * this purpose we call this function from the condition that checks if a wr
5127 * should be skipped, to make sure we don't miss it ( possibly this fmr
5128 * operation was not signalted)
5131 qlnxr_chk_if_fmr(struct qlnxr_qp *qp)
5133 #if __FreeBSD_version >= 1102000
5135 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
5136 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5138 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR)
5139 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5141 #endif /* #if __FreeBSD_version >= 1102000 */
5145 process_req(struct qlnxr_dev *dev,
5146 struct qlnxr_qp *qp,
5147 struct qlnxr_cq *cq,
5151 enum ib_wc_status status,
5155 qlnx_host_t *ha = dev->ha;
5157 QL_DPRINT12(ha, "enter\n");
5159 while (num_entries && qp->sq.wqe_cons != hw_cons) {
5160 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
5161 qlnxr_chk_if_fmr(qp);
5167 wc->status = status;
5170 wc->src_qp = qp->id;
5174 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
5175 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
5177 switch (wc->opcode) {
5178 case IB_WC_RDMA_WRITE:
5180 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
5183 "opcode = IB_WC_RDMA_WRITE bytes = %d\n",
5184 qp->wqe_wr_id[qp->sq.cons].bytes_len);
5187 case IB_WC_COMP_SWAP:
5188 case IB_WC_FETCH_ADD:
5192 #if __FreeBSD_version >= 1102000
5194 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5197 case IB_WC_FAST_REG_MR:
5198 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5200 #endif /* #if __FreeBSD_version >= 1102000 */
5202 case IB_WC_RDMA_READ:
5205 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);
5208 ;//DP_ERR("TBD ERROR");
5215 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
5216 ecore_chain_consume(&qp->sq.pbl);
5217 qlnxr_inc_sw_cons(&qp->sq);
5220 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5225 qlnxr_poll_cq_req(struct qlnxr_dev *dev,
5226 struct qlnxr_qp *qp,
5227 struct qlnxr_cq *cq,
5230 struct rdma_cqe_requester *req)
5233 qlnx_host_t *ha = dev->ha;
5235 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);
5237 switch (req->status) {
5238 case RDMA_CQE_REQ_STS_OK:
5240 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5244 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
5246 if (qp->state != ECORE_ROCE_QP_STATE_ERR)
5247 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5248 IB_WC_WR_FLUSH_ERR, 1);
5251 default: /* other errors case */
5253 /* process all WQE before the cosumer */
5254 qp->state = ECORE_ROCE_QP_STATE_ERR;
5255 cnt = process_req(dev, qp, cq, num_entries, wc,
5256 req->sq_cons - 1, IB_WC_SUCCESS, 0);
5258 /* if we have extra WC fill it with actual error info */
5260 if (cnt < num_entries) {
5261 enum ib_wc_status wc_status;
5263 switch (req->status) {
5264 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
5265 wc_status = IB_WC_BAD_RESP_ERR;
5267 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
5268 wc_status = IB_WC_LOC_LEN_ERR;
5270 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
5271 wc_status = IB_WC_LOC_QP_OP_ERR;
5273 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
5274 wc_status = IB_WC_LOC_PROT_ERR;
5276 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
5277 wc_status = IB_WC_MW_BIND_ERR;
5279 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
5280 wc_status = IB_WC_REM_INV_REQ_ERR;
5282 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
5283 wc_status = IB_WC_REM_ACCESS_ERR;
5285 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
5286 wc_status = IB_WC_REM_OP_ERR;
5288 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
5289 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
5291 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
5292 wc_status = IB_WC_RETRY_EXC_ERR;
5295 wc_status = IB_WC_GENERAL_ERR;
5298 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
5299 wc_status, 1 /* force use of WC */);
5303 QL_DPRINT12(ha, "exit cnt = %d\n", cnt);
5308 __process_resp_one(struct qlnxr_dev *dev,
5309 struct qlnxr_qp *qp,
5310 struct qlnxr_cq *cq,
5312 struct rdma_cqe_responder *resp,
5315 enum ib_wc_status wc_status = IB_WC_SUCCESS;
5316 #if __FreeBSD_version < 1102000
5319 qlnx_host_t *ha = dev->ha;
5321 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",
5324 wc->opcode = IB_WC_RECV;
5327 switch (resp->status) {
5328 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
5329 wc_status = IB_WC_LOC_ACCESS_ERR;
5332 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
5333 wc_status = IB_WC_LOC_LEN_ERR;
5336 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
5337 wc_status = IB_WC_LOC_QP_OP_ERR;
5340 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
5341 wc_status = IB_WC_LOC_PROT_ERR;
5344 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
5345 wc_status = IB_WC_MW_BIND_ERR;
5348 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
5349 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
5352 case RDMA_CQE_RESP_STS_OK:
5354 #if __FreeBSD_version >= 1102000
5355 if (resp->flags & QLNXR_RESP_IMM) {
5357 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5358 wc->wc_flags |= IB_WC_WITH_IMM;
5360 if (resp->flags & QLNXR_RESP_RDMA)
5361 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5363 if (resp->flags & QLNXR_RESP_INV) {
5365 "Invalid flags QLNXR_RESP_INV [0x%x]"
5366 "qp = %p qp->id = 0x%x cq = %p"
5367 " cq->icid = 0x%x\n",
5368 resp->flags, qp, qp->id, cq, cq->icid );
5370 } else if (resp->flags & QLNXR_RESP_INV) {
5372 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5373 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
5375 if (resp->flags & QLNXR_RESP_RDMA) {
5377 "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5378 "qp = %p qp->id = 0x%x cq = %p"
5379 " cq->icid = 0x%x\n",
5380 resp->flags, qp, qp->id, cq, cq->icid );
5382 } else if (resp->flags & QLNXR_RESP_RDMA) {
5383 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5384 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5385 resp->flags, qp, qp->id, cq, cq->icid );
5388 wc_status = IB_WC_SUCCESS;
5389 wc->byte_len = le32_to_cpu(resp->length);
5391 flags = resp->flags & QLNXR_RESP_RDMA_IMM;
5394 case QLNXR_RESP_RDMA_IMM:
5396 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5397 /* fall to set imm data */
5398 case QLNXR_RESP_IMM:
5400 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5401 wc->wc_flags |= IB_WC_WITH_IMM;
5403 case QLNXR_RESP_RDMA:
5404 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5405 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5406 resp->flags, qp, qp->id, cq, cq->icid );
5409 /* valid configuration, but nothing todo here */
5412 #endif /* #if __FreeBSD_version >= 1102000 */
5416 wc_status = IB_WC_GENERAL_ERR;
5420 wc->status = wc_status;
5422 wc->src_qp = qp->id;
5426 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);
5432 process_resp_one_srq(struct qlnxr_dev *dev,
5433 struct qlnxr_qp *qp,
5434 struct qlnxr_cq *cq,
5436 struct rdma_cqe_responder *resp)
5438 struct qlnxr_srq *srq = qp->srq;
5440 qlnx_host_t *ha = dev->ha;
5442 QL_DPRINT12(ha, "enter\n");
5444 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);
5446 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5447 wc->status = IB_WC_WR_FLUSH_ERR;
5451 wc->src_qp = qp->id;
5455 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5458 /* PBL is maintained in case of WR granularity.
5459 * So increment WR consumer after consuming WR
5461 srq->hw_srq.wr_cons_cnt++;
5463 QL_DPRINT12(ha, "exit\n");
5468 process_resp_one(struct qlnxr_dev *dev,
5469 struct qlnxr_qp *qp,
5470 struct qlnxr_cq *cq,
5472 struct rdma_cqe_responder *resp)
5474 qlnx_host_t *ha = dev->ha;
5475 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5477 QL_DPRINT12(ha, "enter\n");
5479 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5481 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5482 ecore_chain_consume(&qp->rq.pbl);
5483 qlnxr_inc_sw_cons(&qp->rq);
5485 QL_DPRINT12(ha, "exit\n");
5490 process_resp_flush(struct qlnxr_qp *qp,
5496 qlnx_host_t *ha = qp->dev->ha;
5498 QL_DPRINT12(ha, "enter\n");
5500 while (num_entries && qp->rq.wqe_cons != hw_cons) {
5502 wc->status = IB_WC_WR_FLUSH_ERR;
5505 wc->src_qp = qp->id;
5507 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5512 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5513 ecore_chain_consume(&qp->rq.pbl);
5514 qlnxr_inc_sw_cons(&qp->rq);
5517 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5522 try_consume_resp_cqe(struct qlnxr_cq *cq,
5523 struct qlnxr_qp *qp,
5524 struct rdma_cqe_responder *resp,
5527 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
5534 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev,
5535 struct qlnxr_qp *qp,
5536 struct qlnxr_cq *cq,
5539 struct rdma_cqe_responder *resp,
5543 qlnx_host_t *ha = dev->ha;
5545 QL_DPRINT12(ha, "enter\n");
5547 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
5551 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5556 qlnxr_poll_cq_resp(struct qlnxr_dev *dev,
5557 struct qlnxr_qp *qp,
5558 struct qlnxr_cq *cq,
5561 struct rdma_cqe_responder *resp,
5565 qlnx_host_t *ha = dev->ha;
5567 QL_DPRINT12(ha, "enter\n");
5569 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5570 cnt = process_resp_flush(qp, num_entries, wc,
5572 try_consume_resp_cqe(cq, qp, resp, update);
5574 cnt = process_resp_one(dev, qp, cq, wc, resp);
5579 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5584 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp,
5585 struct rdma_cqe_requester *req, int *update)
5587 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
5594 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags)
5597 qlnx_host_t *ha = dev->ha;
5599 QL_DPRINT12(ha, "enter\n");
5602 cq->db.data.agg_flags = flags;
5603 cq->db.data.value = cpu_to_le32(cons);
5605 reg_addr = (uint64_t)((uint8_t *)cq->db_addr -
5606 (uint8_t *)(ha->cdev.doorbells));
5608 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);
5609 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5611 QL_DPRINT12(ha, "exit\n");
5615 // writeq(cq->db.raw, cq->db_addr);
5617 /* Note that since the FW allows 64 bit write only, in 32bit systems
5618 * the value of db_addr must be low enough. This is currently not
5621 // writel(cq->db.raw & 0xffffffff, cq->db_addr);
5627 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe)
5629 struct rdma_cqe_requester *resp_cqe = &cqe->req;
5630 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==
5635 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
5637 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5638 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
5640 union rdma_cqe *cqe = cq->latest_cqe;
5642 u32 old_cons, new_cons;
5643 unsigned long flags;
5644 qlnx_host_t *ha = dev->ha;
5646 QL_DPRINT12(ha, "enter\n");
5648 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5651 if (cq->destroyed) {
5652 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n",
5657 if (cq->cq_type == QLNXR_CQ_TYPE_GSI)
5658 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc);
5660 spin_lock_irqsave(&cq->cq_lock, flags);
5662 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5664 while (num_entries && is_valid_cqe(cq, cqe)) {
5666 struct qlnxr_qp *qp;
5667 struct rdma_cqe_requester *resp_cqe;
5668 enum rdma_cqe_type cqe_type;
5670 /* prevent speculative reads of any field of CQE */
5673 resp_cqe = &cqe->req;
5674 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,
5675 resp_cqe->qp_handle.lo);
5678 QL_DPRINT11(ha, "qp = NULL\n");
5684 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
5687 case RDMA_CQE_TYPE_REQUESTER:
5688 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries,
5690 try_consume_req_cqe(cq, qp, &cqe->req, &update);
5692 case RDMA_CQE_TYPE_RESPONDER_RQ:
5693 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries,
5694 wc, &cqe->resp, &update);
5696 case RDMA_CQE_TYPE_RESPONDER_SRQ:
5697 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries,
5698 wc, &cqe->resp, &update);
5700 case RDMA_CQE_TYPE_INVALID:
5702 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);
5709 cqe = cq->latest_cqe;
5711 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5713 cq->cq_cons += new_cons - old_cons;
5716 /* doorbell notifies abount latest VALID entry,
5717 * but chain already point to the next INVALID one
5719 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);
5720 QL_DPRINT12(ha, "cq = %p cons = 0x%x "
5721 "arm_flags = 0x%x db.icid = 0x%x\n", cq,
5722 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);
5725 spin_unlock_irqrestore(&cq->cq_lock, flags);
5727 QL_DPRINT12(ha, "exit\n");
5733 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
5735 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5736 unsigned long sflags;
5737 struct qlnxr_dev *dev;
5740 dev = get_qlnxr_dev((ibcq->device));
5743 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "
5744 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,
5745 flags, cq, cq->cq_cons, cq->cq_type);
5747 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5750 if (cq->destroyed) {
5751 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n",
5756 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {
5760 spin_lock_irqsave(&cq->cq_lock, sflags);
5764 if (flags & IB_CQ_SOLICITED) {
5765 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
5767 if (flags & IB_CQ_NEXT_COMP) {
5768 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
5771 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);
5773 spin_unlock_irqrestore(&cq->cq_lock, sflags);
5775 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);
5779 static struct qlnxr_mr *
5780 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
5782 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
5783 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
5784 struct qlnxr_mr *mr;
5790 QL_DPRINT12(ha, "enter ibpd = %p pd = %p "
5791 " pd_id = %d max_page_list_len = %d\n",
5792 ibpd, pd, pd->pd_id, max_page_list_len);
5794 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
5796 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
5801 mr->type = QLNXR_MR_FRMR;
5803 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,
5804 1 /* allow dual layer pbl */);
5806 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n");
5810 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
5812 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n");
5816 /* index only, 18 bit long, lkey = itid << 8 | key */
5817 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;
5819 mr->hw_mr.pd = pd->pd_id;
5820 mr->hw_mr.local_read = 1;
5821 mr->hw_mr.local_write = 0;
5822 mr->hw_mr.remote_read = 0;
5823 mr->hw_mr.remote_write = 0;
5824 mr->hw_mr.remote_atomic = 0;
5825 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
5826 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */
5827 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
5828 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
5830 mr->hw_mr.length = 0;
5831 mr->hw_mr.vaddr = 0;
5832 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
5833 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */
5834 mr->hw_mr.dma_mr = false;
5836 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
5838 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n");
5842 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
5843 mr->ibmr.rkey = mr->ibmr.lkey;
5845 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",
5851 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
5855 QL_DPRINT12(ha, "exit\n");
5860 #if __FreeBSD_version >= 1102000
5863 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
5865 struct qlnxr_dev *dev;
5866 struct qlnxr_mr *mr;
5869 dev = get_qlnxr_dev(ibpd->device);
5872 QL_DPRINT12(ha, "enter\n");
5874 if (mr_type != IB_MR_TYPE_MEM_REG)
5875 return ERR_PTR(-EINVAL);
5877 mr = __qlnxr_alloc_mr(ibpd, max_num_sg);
5880 return ERR_PTR(-EINVAL);
5882 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);
5888 qlnxr_set_page(struct ib_mr *ibmr, u64 addr)
5890 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5891 struct qlnxr_pbl *pbl_table;
5892 struct regpair *pbe;
5893 struct qlnxr_dev *dev;
5900 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
5901 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);
5905 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,
5908 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
5909 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
5910 pbe = (struct regpair *)pbl_table->va;
5911 pbe += mr->npages % pbes_in_page;
5912 pbe->lo = cpu_to_le32((u32)addr);
5913 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
5917 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,
5923 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
5924 int sg_nents, unsigned int *sg_offset)
5927 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5933 if (mr->dev == NULL)
5938 QL_DPRINT12(ha, "enter\n");
5941 qlnx_handle_completed_mrs(mr->dev, &mr->info);
5943 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page);
5945 QL_DPRINT12(ha, "exit ret = %d\n", ret);
5953 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
5955 struct qlnxr_dev *dev;
5956 struct qlnxr_mr *mr;
5958 struct ib_mr *ibmr = NULL;
5960 dev = get_qlnxr_dev((ibpd->device));
5963 QL_DPRINT12(ha, "enter\n");
5965 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len);
5968 ibmr = ERR_PTR(-EINVAL);
5973 QL_DPRINT12(ha, "exit %p\n", ibmr);
5978 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
5980 struct qlnxr_fast_reg_page_list *frmr_list;
5982 frmr_list = get_qlnxr_frmr_list(page_list);
5984 free_mr_info(frmr_list->dev, &frmr_list->info);
5986 kfree(frmr_list->ibfrpl.page_list);
5992 struct ib_fast_reg_page_list *
5993 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len)
5995 struct qlnxr_fast_reg_page_list *frmr_list = NULL;
5996 struct qlnxr_dev *dev;
5997 int size = page_list_len * sizeof(u64);
6001 dev = get_qlnxr_dev(ibdev);
6004 QL_DPRINT12(ha, "enter\n");
6006 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL);
6008 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n");
6012 frmr_list->dev = dev;
6013 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL);
6014 if (!frmr_list->ibfrpl.page_list) {
6015 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n");
6019 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len,
6020 1 /* allow dual layer pbl */);
6024 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl);
6026 return &frmr_list->ibfrpl;
6029 kfree(frmr_list->ibfrpl.page_list);
6033 QL_DPRINT12(ha, "exit with error\n");
6039 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6040 int buf_cnt, uint64_t *total_size)
6046 if (!buf_cnt || buf_list == NULL) {
6048 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt);
6052 size = buf_list->size;
6056 "failed buf_list = %p buf_cnt = %d"
6057 " buf_list->size = 0\n", buf_list, buf_cnt);
6062 *total_size += buf_list->size;
6064 if (buf_list->size != size) {
6066 "failed buf_list = %p buf_cnt = %d"
6067 " all buffers should have same size\n",
6079 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6083 size_t num_pages = 0;
6086 for (i = 0; i < buf_cnt; i++) {
6088 while (size < buf_list->size) {
6098 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev,
6099 struct ib_phys_buf *buf_list, int buf_cnt,
6100 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
6102 struct regpair *pbe;
6103 struct qlnxr_pbl *pbl_tbl;
6104 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
6111 QL_DPRINT12(ha, "enter\n");
6114 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
6118 if (!pbl_info->num_pbes) {
6119 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
6123 /* If we have a two layered pbl, the first pbl points to the rest
6124 * of the pbls and the first entry lays on the second pbl in the table
6126 if (pbl_info->two_layered)
6131 pbe = (struct regpair *)pbl_tbl->va;
6133 QL_DPRINT12(ha, "pbe is NULL\n");
6139 for (i = 0; i < buf_cnt; i++) {
6140 pages = buf_list->size >> PAGE_SHIFT;
6142 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
6143 /* store the page address in pbe */
6145 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt);
6147 pbe->lo = cpu_to_le32((u32)pbe_addr);
6148 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32)));
6150 QL_DPRINT12(ha, "Populate pbl table:"
6151 " pbe->addr=0x%x:0x%x "
6152 " pbe_cnt = %d total_num_pbes=%d"
6153 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
6154 total_num_pbes, pbe);
6160 if (total_num_pbes == pbl_info->num_pbes)
6163 /* if the given pbl is full storing the pbes,
6164 * move to next pbl. */
6166 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
6168 pbe = (struct regpair *)pbl_tbl->va;
6174 QL_DPRINT12(ha, "exit\n");
6179 qlnxr_reg_kernel_mr(struct ib_pd *ibpd,
6180 struct ib_phys_buf *buf_list,
6181 int buf_cnt, int acc, u64 *iova_start)
6184 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
6185 struct qlnxr_mr *mr;
6186 struct qlnxr_pd *pd;
6188 size_t num_pages = 0;
6193 QL_DPRINT12(ha, "enter\n");
6195 pd = get_qlnxr_pd(ibpd);
6197 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d,"
6198 " iova_start = %p, acc = %d\n",
6199 pd->pd_id, buf_list, buf_cnt, iova_start, acc);
6201 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
6202 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&"
6203 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
6204 // return ERR_PTR(-EINVAL);
6207 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
6209 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
6213 mr->type = QLNXR_MR_KERNEL;
6214 mr->iova_start = iova_start;
6216 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length);
6220 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt);
6224 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1);
6227 "qlnxr_init_mr_info failed [%d]\n", rc);
6231 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table,
6232 &mr->info.pbl_info);
6234 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
6237 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
6241 /* index only, 18 bit long, lkey = itid << 8 | key */
6242 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
6244 mr->hw_mr.pd = pd->pd_id;
6245 mr->hw_mr.local_read = 1;
6246 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
6247 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
6248 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
6249 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
6250 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
6251 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
6252 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
6253 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
6254 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */
6258 mr->hw_mr.length = length;
6259 mr->hw_mr.vaddr = (uint64_t)iova_start;
6260 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
6261 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
6262 mr->hw_mr.dma_mr = false;
6264 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
6266 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
6270 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6271 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
6272 mr->hw_mr.remote_atomic)
6273 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6275 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey);
6280 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
6282 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
6286 QL_DPRINT12(ha, "exit [%d]\n", rc);
6287 return (ERR_PTR(rc));
6290 #endif /* #if __FreeBSD_version >= 1102000 */
6293 #if __FreeBSD_version >= 1102000
6294 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
6295 struct ib_udata *udata)
6297 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
6298 #endif /* #if __FreeBSD_version >= 1102000 */
6300 struct qlnxr_dev *dev;
6302 struct qlnxr_ah *ah;
6304 dev = get_qlnxr_dev((ibpd->device));
6307 QL_DPRINT12(ha, "in create_ah\n");
6309 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
6311 QL_DPRINT12(ha, "no address handle can be allocated\n");
6312 return ERR_PTR(-ENOMEM);
6321 qlnxr_destroy_ah(struct ib_ah *ibah)
6323 struct qlnxr_dev *dev;
6325 struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
6327 dev = get_qlnxr_dev((ibah->device));
6330 QL_DPRINT12(ha, "in destroy_ah\n");
6337 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6339 struct qlnxr_dev *dev;
6342 dev = get_qlnxr_dev((ibah->device));
6344 QL_DPRINT12(ha, "Query AH not supported\n");
6349 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6351 struct qlnxr_dev *dev;
6354 dev = get_qlnxr_dev((ibah->device));
6356 QL_DPRINT12(ha, "Modify AH not supported\n");
6360 #if __FreeBSD_version >= 1102000
6362 qlnxr_process_mad(struct ib_device *ibdev,
6363 int process_mad_flags,
6365 const struct ib_wc *in_wc,
6366 const struct ib_grh *in_grh,
6367 const struct ib_mad_hdr *mad_hdr,
6369 struct ib_mad_hdr *out_mad,
6370 size_t *out_mad_size,
6371 u16 *out_mad_pkey_index)
6376 qlnxr_process_mad(struct ib_device *ibdev,
6377 int process_mad_flags,
6379 struct ib_wc *in_wc,
6380 struct ib_grh *in_grh,
6381 struct ib_mad *in_mad,
6382 struct ib_mad *out_mad)
6384 #endif /* #if __FreeBSD_version >= 1102000 */
6386 struct qlnxr_dev *dev;
6389 dev = get_qlnxr_dev(ibdev);
6391 QL_DPRINT12(ha, "process mad not supported\n");
6394 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",
6395 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,
6396 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,
6397 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,
6398 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);
6400 // return IB_MAD_RESULT_SUCCESS;
6403 #if __FreeBSD_version >= 1102000
6405 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
6406 struct ib_port_immutable *immutable)
6408 struct qlnxr_dev *dev;
6410 struct ib_port_attr attr;
6413 dev = get_qlnxr_dev(ibdev);
6416 QL_DPRINT12(ha, "enter\n");
6418 err = qlnxr_query_port(ibdev, port_num, &attr);
6422 if (QLNX_IS_IWARP(dev)) {
6423 immutable->pkey_tbl_len = 1;
6424 immutable->gid_tbl_len = 1;
6425 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
6426 immutable->max_mad_size = 0;
6428 immutable->pkey_tbl_len = attr.pkey_tbl_len;
6429 immutable->gid_tbl_len = attr.gid_tbl_len;
6430 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
6431 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
6434 QL_DPRINT12(ha, "exit\n");
6437 #endif /* #if __FreeBSD_version > 1102000 */
6439 /***** iWARP related functions *************/
6442 qlnxr_iw_mpa_request(void *context,
6443 struct ecore_iwarp_cm_event_params *params)
6445 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context;
6446 struct qlnxr_dev *dev = listener->dev;
6447 struct qlnxr_iw_ep *ep;
6448 struct iw_cm_event event;
6449 struct sockaddr_in *laddr;
6450 struct sockaddr_in *raddr;
6455 QL_DPRINT12(ha, "enter\n");
6457 if (params->cm_info->ip_version != ECORE_TCP_IPV4) {
6458 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",
6459 params->cm_info->ip_version);
6463 ep = kzalloc(sizeof(*ep), GFP_ATOMIC);
6466 QL_DPRINT11(ha, "kzalloc{ep) failed\n");
6471 ep->ecore_context = params->ep_context;
6473 memset(&event, 0, sizeof(event));
6475 event.event = IW_CM_EVENT_CONNECT_REQUEST;
6476 event.status = params->status;
6478 laddr = (struct sockaddr_in *)&event.local_addr;
6479 raddr = (struct sockaddr_in *)&event.remote_addr;
6481 laddr->sin_family = AF_INET;
6482 raddr->sin_family = AF_INET;
6484 laddr->sin_port = htons(params->cm_info->local_port);
6485 raddr->sin_port = htons(params->cm_info->remote_port);
6487 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);
6488 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);
6490 event.provider_data = (void *)ep;
6491 event.private_data = (void *)params->cm_info->private_data;
6492 event.private_data_len = (u8)params->cm_info->private_data_len;
6494 #if __FreeBSD_version >= 1100000
6495 event.ord = params->cm_info->ord;
6496 event.ird = params->cm_info->ird;
6497 #endif /* #if __FreeBSD_version >= 1100000 */
6499 listener->cm_id->event_handler(listener->cm_id, &event);
6501 QL_DPRINT12(ha, "exit\n");
6507 qlnxr_iw_issue_event(void *context,
6508 struct ecore_iwarp_cm_event_params *params,
6509 enum iw_cm_event_type event_type,
6512 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6513 struct qlnxr_dev *dev = ep->dev;
6514 struct iw_cm_event event;
6519 QL_DPRINT12(ha, "enter\n");
6521 memset(&event, 0, sizeof(event));
6522 event.status = params->status;
6523 event.event = event_type;
6525 if (params->cm_info != NULL) {
6526 #if __FreeBSD_version >= 1100000
6527 event.ird = params->cm_info->ird;
6528 event.ord = params->cm_info->ord;
6529 QL_DPRINT12(ha, "ord=[%d] \n", event.ord);
6530 QL_DPRINT12(ha, "ird=[%d] \n", event.ird);
6531 #endif /* #if __FreeBSD_version >= 1100000 */
6533 event.private_data_len = params->cm_info->private_data_len;
6534 event.private_data = (void *)params->cm_info->private_data;
6535 QL_DPRINT12(ha, "private_data_len=[%d] \n",
6536 event.private_data_len);
6539 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str);
6540 QL_DPRINT12(ha, "status=[%d] \n", event.status);
6544 ep->cm_id->event_handler(ep->cm_id, &event);
6546 QL_DPRINT11(ha, "ep->cm_id == NULL \n");
6548 QL_DPRINT11(ha, "ep == NULL \n");
6551 QL_DPRINT12(ha, "exit\n");
6557 qlnxr_iw_close_event(void *context,
6558 struct ecore_iwarp_cm_event_params *params)
6560 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6561 struct qlnxr_dev *dev = ep->dev;
6566 QL_DPRINT12(ha, "enter\n");
6569 qlnxr_iw_issue_event(context,
6572 "IW_CM_EVENT_EVENT_CLOSE");
6573 ep->cm_id->rem_ref(ep->cm_id);
6577 QL_DPRINT12(ha, "exit\n");
6582 #if __FreeBSD_version >= 1102000
6585 qlnxr_iw_passive_complete(void *context,
6586 struct ecore_iwarp_cm_event_params *params)
6588 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6589 struct qlnxr_dev *dev = ep->dev;
6594 /* We will only reach the following state if MPA_REJECT was called on
6595 * passive. In this case there will be no associated QP.
6597 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {
6598 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n");
6603 /* We always issue an established event, however, ofed does not look
6604 * at event code for established. So if there was a failure, we follow
6607 qlnxr_iw_issue_event(context,
6609 IW_CM_EVENT_ESTABLISHED,
6610 "IW_CM_EVENT_ESTABLISHED");
6612 if (params->status < 0) {
6613 qlnxr_iw_close_event(context, params);
6619 struct qlnxr_discon_work {
6620 struct work_struct work;
6621 struct qlnxr_iw_ep *ep;
6622 enum ecore_iwarp_event_type event;
6627 qlnxr_iw_disconnect_worker(struct work_struct *work)
6629 struct qlnxr_discon_work *dwork =
6630 container_of(work, struct qlnxr_discon_work, work);
6631 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
6632 struct qlnxr_iw_ep *ep = dwork->ep;
6633 struct qlnxr_dev *dev = ep->dev;
6634 struct qlnxr_qp *qp = ep->qp;
6635 struct iw_cm_event event;
6637 if (qp->destroyed) {
6639 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6643 memset(&event, 0, sizeof(event));
6644 event.status = dwork->status;
6645 event.event = IW_CM_EVENT_DISCONNECT;
6647 /* Success means graceful disconnect was requested. modifying
6648 * to SQD is translated to graceful disconnect. O/w reset is sent
6651 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR;
6653 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD;
6658 ep->cm_id->event_handler(ep->cm_id, &event);
6660 SET_FIELD(qp_params.modify_flags,
6661 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
6663 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
6665 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6671 qlnxr_iw_disconnect_event(void *context,
6672 struct ecore_iwarp_cm_event_params *params)
6674 struct qlnxr_discon_work *work;
6675 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6676 struct qlnxr_dev *dev = ep->dev;
6677 struct qlnxr_qp *qp = ep->qp;
6679 work = kzalloc(sizeof(*work), GFP_ATOMIC);
6683 qlnxr_iw_qp_add_ref(&qp->ibqp);
6685 work->event = params->event;
6686 work->status = params->status;
6688 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);
6689 queue_work(dev->iwarp_wq, &work->work);
6694 #endif /* #if __FreeBSD_version >= 1102000 */
6697 qlnxr_iw_mpa_reply(void *context,
6698 struct ecore_iwarp_cm_event_params *params)
6700 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6701 struct qlnxr_dev *dev = ep->dev;
6702 struct ecore_iwarp_send_rtr_in rtr_in;
6708 QL_DPRINT12(ha, "enter\n");
6710 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6713 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in));
6714 rtr_in.ep_context = params->ep_context;
6716 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);
6718 QL_DPRINT12(ha, "exit rc = %d\n", rc);
6723 qlnxr_iw_qp_event(void *context,
6724 struct ecore_iwarp_cm_event_params *params,
6725 enum ib_event_type ib_event,
6728 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6729 struct qlnxr_dev *dev = ep->dev;
6730 struct ib_qp *ibqp = &(ep->qp->ibqp);
6731 struct ib_event event;
6737 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",
6738 context, params->event, str, ibqp->event_handler);
6740 if (ibqp->event_handler) {
6741 event.event = ib_event;
6742 event.device = ibqp->device;
6743 event.element.qp = ibqp;
6744 ibqp->event_handler(&event, ibqp->qp_context);
6751 qlnxr_iw_event_handler(void *context,
6752 struct ecore_iwarp_cm_event_params *params)
6754 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6755 struct qlnxr_dev *dev = ep->dev;
6760 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6761 "enter\n", context, params->event);
6763 switch (params->event) {
6764 /* Passive side request received */
6765 case ECORE_IWARP_EVENT_MPA_REQUEST:
6766 qlnxr_iw_mpa_request(context, params);
6769 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY:
6770 qlnxr_iw_mpa_reply(context, params);
6773 /* Passive side established ( ack on mpa response ) */
6774 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE:
6776 #if __FreeBSD_version >= 1102000
6778 ep->during_connect = 0;
6779 qlnxr_iw_passive_complete(context, params);
6782 qlnxr_iw_issue_event(context,
6784 IW_CM_EVENT_ESTABLISHED,
6785 "IW_CM_EVENT_ESTABLISHED");
6786 #endif /* #if __FreeBSD_version >= 1102000 */
6789 /* Active side reply received */
6790 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE:
6791 ep->during_connect = 0;
6792 qlnxr_iw_issue_event(context,
6794 IW_CM_EVENT_CONNECT_REPLY,
6795 "IW_CM_EVENT_CONNECT_REPLY");
6796 if (params->status < 0) {
6797 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6799 ep->cm_id->rem_ref(ep->cm_id);
6804 case ECORE_IWARP_EVENT_DISCONNECT:
6806 #if __FreeBSD_version >= 1102000
6807 qlnxr_iw_disconnect_event(context, params);
6809 qlnxr_iw_issue_event(context,
6811 IW_CM_EVENT_DISCONNECT,
6812 "IW_CM_EVENT_DISCONNECT");
6813 qlnxr_iw_close_event(context, params);
6814 #endif /* #if __FreeBSD_version >= 1102000 */
6817 case ECORE_IWARP_EVENT_CLOSE:
6818 ep->during_connect = 0;
6819 qlnxr_iw_close_event(context, params);
6822 case ECORE_IWARP_EVENT_RQ_EMPTY:
6823 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6824 "IWARP_EVENT_RQ_EMPTY");
6827 case ECORE_IWARP_EVENT_IRQ_FULL:
6828 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6829 "IWARP_EVENT_IRQ_FULL");
6832 case ECORE_IWARP_EVENT_LLP_TIMEOUT:
6833 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6834 "IWARP_EVENT_LLP_TIMEOUT");
6837 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR:
6838 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6839 "IWARP_EVENT_REMOTE_PROTECTION_ERROR");
6842 case ECORE_IWARP_EVENT_CQ_OVERFLOW:
6843 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6844 "QED_IWARP_EVENT_CQ_OVERFLOW");
6847 case ECORE_IWARP_EVENT_QP_CATASTROPHIC:
6848 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6849 "QED_IWARP_EVENT_QP_CATASTROPHIC");
6852 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR:
6853 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6854 "IWARP_EVENT_LOCAL_ACCESS_ERROR");
6857 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR:
6858 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6859 "IWARP_EVENT_REMOTE_OPERATION_ERROR");
6862 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED:
6863 QL_DPRINT12(ha, "Got terminate message"
6864 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n");
6869 "Unknown event [0x%x] received \n", params->event);
6873 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6874 "exit\n", context, params->event);
6879 qlnxr_addr4_resolve(struct qlnxr_dev *dev,
6880 struct sockaddr_in *src_in,
6881 struct sockaddr_in *dst_in,
6886 #if __FreeBSD_version >= 1100000
6887 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,
6888 dst_mac, NULL, NULL);
6890 struct llentry *lle;
6892 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in,
6896 QL_DPRINT12(dev->ha, "rc = %d "
6897 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "
6898 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,
6899 dst_in->sin_len, dst_in->sin_family,
6900 NIPQUAD((dst_in->sin_addr.s_addr)),
6901 dst_mac[0], dst_mac[1], dst_mac[2],
6902 dst_mac[3], dst_mac[4], dst_mac[5]);
6908 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
6910 struct qlnxr_dev *dev;
6911 struct ecore_iwarp_connect_out out_params;
6912 struct ecore_iwarp_connect_in in_params;
6913 struct qlnxr_iw_ep *ep;
6914 struct qlnxr_qp *qp;
6915 struct sockaddr_in *laddr;
6916 struct sockaddr_in *raddr;
6920 dev = get_qlnxr_dev((cm_id->device));
6923 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] "
6924 "enter \n", cm_id, conn_param);
6926 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6929 qp = idr_find(&dev->qpidr, conn_param->qpn);
6931 laddr = (struct sockaddr_in *)&cm_id->local_addr;
6932 raddr = (struct sockaddr_in *)&cm_id->remote_addr;
6935 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n",
6936 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,
6937 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);
6939 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
6941 QL_DPRINT11(ha, "struct qlnxr_iw_ep "
6942 "alloc memory failed\n");
6948 cm_id->add_ref(cm_id);
6951 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in));
6952 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out));
6954 in_params.event_cb = qlnxr_iw_event_handler;
6955 in_params.cb_context = ep;
6957 in_params.cm_info.ip_version = ECORE_TCP_IPV4;
6959 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);
6960 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);
6961 in_params.cm_info.remote_port = ntohs(raddr->sin_port);
6962 in_params.cm_info.local_port = ntohs(laddr->sin_port);
6963 in_params.cm_info.vlan = 0;
6964 in_params.mss = dev->ha->ifp->if_mtu - 40;
6966 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] "
6967 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d "
6969 NIPQUAD((in_params.cm_info.remote_ip[0])),
6970 NIPQUAD((in_params.cm_info.local_ip[0])),
6971 in_params.cm_info.remote_port, in_params.cm_info.local_port,
6972 in_params.cm_info.vlan);
6974 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);
6977 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n");
6981 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p"
6982 " private_data_len=%d rq_psn=%d\n",
6983 conn_param->ord, conn_param->ird, conn_param->private_data,
6984 conn_param->private_data_len, qp->rq_psn);
6986 in_params.cm_info.ord = conn_param->ord;
6987 in_params.cm_info.ird = conn_param->ird;
6988 in_params.cm_info.private_data = conn_param->private_data;
6989 in_params.cm_info.private_data_len = conn_param->private_data_len;
6990 in_params.qp = qp->ecore_qp;
6992 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
6994 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);
6997 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n");
7001 QL_DPRINT12(ha, "exit\n");
7006 cm_id->rem_ref(cm_id);
7009 QL_DPRINT12(ha, "exit [%d]\n", rc);
7014 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
7016 struct qlnxr_dev *dev;
7017 struct qlnxr_iw_listener *listener;
7018 struct ecore_iwarp_listen_in iparams;
7019 struct ecore_iwarp_listen_out oparams;
7020 struct sockaddr_in *laddr;
7024 dev = get_qlnxr_dev((cm_id->device));
7027 QL_DPRINT12(ha, "enter\n");
7029 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7032 laddr = (struct sockaddr_in *)&cm_id->local_addr;
7034 listener = kzalloc(sizeof(*listener), GFP_KERNEL);
7036 if (listener == NULL) {
7037 QL_DPRINT11(ha, "listener memory alloc failed\n");
7041 listener->dev = dev;
7042 cm_id->add_ref(cm_id);
7043 listener->cm_id = cm_id;
7044 listener->backlog = backlog;
7046 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in));
7047 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out));
7049 iparams.cb_context = listener;
7050 iparams.event_cb = qlnxr_iw_event_handler;
7051 iparams.max_backlog = backlog;
7053 iparams.ip_version = ECORE_TCP_IPV4;
7055 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);
7056 iparams.port = ntohs(laddr->sin_port);
7059 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n",
7060 NIPQUAD((laddr->sin_addr.s_addr)),
7061 laddr->sin_port, iparams.port);
7063 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);
7066 "ecore_iwarp_create_listen failed rc = %d\n", rc);
7070 listener->ecore_handle = oparams.handle;
7071 cm_id->provider_data = listener;
7073 QL_DPRINT12(ha, "exit\n");
7077 cm_id->rem_ref(cm_id);
7080 QL_DPRINT12(ha, "exit [%d]\n", rc);
7085 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
7087 struct qlnxr_iw_listener *listener = cm_id->provider_data;
7088 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));
7094 QL_DPRINT12(ha, "enter\n");
7096 if (listener->ecore_handle)
7097 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,
7098 listener->ecore_handle);
7100 cm_id->rem_ref(cm_id);
7102 QL_DPRINT12(ha, "exit [%d]\n", rc);
7107 qlnxr_iw_accept(struct iw_cm_id *cm_id,
7108 struct iw_cm_conn_param *conn_param)
7110 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7111 struct qlnxr_dev *dev = ep->dev;
7112 struct qlnxr_qp *qp;
7113 struct ecore_iwarp_accept_in params;
7119 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn);
7121 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7124 qp = idr_find(&dev->qpidr, conn_param->qpn);
7126 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n",
7132 cm_id->add_ref(cm_id);
7135 params.ep_context = ep->ecore_context;
7136 params.cb_context = ep;
7137 params.qp = ep->qp->ecore_qp;
7138 params.private_data = conn_param->private_data;
7139 params.private_data_len = conn_param->private_data_len;
7140 params.ird = conn_param->ird;
7141 params.ord = conn_param->ord;
7143 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms);
7145 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);
7149 QL_DPRINT12(ha, "exit\n");
7152 cm_id->rem_ref(cm_id);
7153 QL_DPRINT12(ha, "exit rc = %d\n", rc);
7158 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
7160 #if __FreeBSD_version >= 1102000
7162 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7163 struct qlnxr_dev *dev = ep->dev;
7164 struct ecore_iwarp_reject_in params;
7167 params.ep_context = ep->ecore_context;
7168 params.cb_context = ep;
7169 params.private_data = pdata;
7170 params.private_data_len = pdata_len;
7173 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms);
7179 printf("iWARP reject_cr not implemented\n");
7182 #endif /* #if __FreeBSD_version >= 1102000 */
7186 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp)
7188 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7193 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp);
7195 atomic_inc(&qp->refcnt);
7197 QL_DPRINT12(ha, "exit \n");
7202 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
7204 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7209 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp);
7211 if (atomic_dec_and_test(&qp->refcnt)) {
7212 qlnxr_idr_remove(qp->dev, qp->qp_id);
7216 QL_DPRINT12(ha, "exit \n");
7221 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn)
7223 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
7229 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn);
7231 qp = idr_find(&dev->qpidr, qpn);
7233 QL_DPRINT12(ha, "exit qp = %p\n", qp);