2 * Copyright (c) 2018-2019 Cavium, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include "qlnxr_def.h"
36 #include "rdma_common.h"
37 #include "qlnxr_roce.h"
40 #define upper_32_bits(x) (uint32_t)(x >> 32)
41 #define lower_32_bits(x) (uint32_t)(x)
42 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
44 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \
46 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
47 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
51 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
53 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
54 (sge)->length = cpu_to_le32(vlength); \
55 (sge)->flags = cpu_to_le32(vflags); \
58 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
60 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \
61 (hdr)->num_sges = num_sge; \
64 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
66 TYPEPTR_ADDR_SET(sge, addr, vaddr); \
67 (sge)->length = cpu_to_le32(vlength); \
68 (sge)->l_key = cpu_to_le32(vlkey); \
71 #define NIPQUAD(addr) \
72 ((unsigned char *)&addr)[0], \
73 ((unsigned char *)&addr)[1], \
74 ((unsigned char *)&addr)[2], \
75 ((unsigned char *)&addr)[3]
78 qlnxr_check_srq_params(struct ib_pd *ibpd,
79 struct qlnxr_dev *dev,
80 struct ib_srq_init_attr *attrs);
83 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
84 struct qlnxr_srq *srq,
85 struct qlnxr_create_srq_ureq *ureq,
86 int access, int dmasync);
89 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
90 struct qlnxr_dev *dev,
91 struct ib_srq_init_attr *init_attr);
95 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
96 struct qlnxr_srq *srq,
97 struct ib_udata *udata);
100 qlnxr_free_srq_user_params(struct qlnxr_srq *srq);
103 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq);
107 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq);
110 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index,
113 struct qlnxr_dev *dev;
116 dev = get_qlnxr_dev(ibdev);
119 QL_DPRINT12(ha, "enter\n");
121 memset(sgid->raw, 0, sizeof(sgid->raw));
123 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));
125 QL_DPRINT12(ha, "exit\n");
131 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
134 struct qlnxr_dev *dev;
137 dev = get_qlnxr_dev(ibdev);
139 QL_DPRINT12(ha, "enter index: %d\n", index);
142 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */
143 //if (!rdma_cap_roce_gid_table(ibdev, port)) {
144 if (!(rdma_protocol_roce(ibdev, port) &&
145 ibdev->add_gid && ibdev->del_gid)) {
146 QL_DPRINT11(ha, "acquire gid failed\n");
150 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
151 if (ret == -EAGAIN) {
152 memcpy(sgid, &zgid, sizeof(*sgid));
156 if ((index >= QLNXR_MAX_SGID) || (index < 0)) {
157 QL_DPRINT12(ha, "invalid gid index %d\n", index);
158 memset(sgid, 0, sizeof(*sgid));
161 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
163 QL_DPRINT12(ha, "exit : %p\n", sgid);
169 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
170 struct ib_udata *udata)
172 struct qlnxr_dev *dev;
174 struct ecore_rdma_destroy_srq_in_params destroy_in_params;
175 struct ecore_rdma_create_srq_out_params out_params;
176 struct ecore_rdma_create_srq_in_params in_params;
177 u64 pbl_base_addr, phy_prod_pair_addr;
178 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
179 struct ib_ucontext *ib_ctx = NULL;
180 struct qlnxr_srq_hwq_info *hw_srq;
181 struct qlnxr_ucontext *ctx = NULL;
182 struct qlnxr_create_srq_ureq ureq;
183 u32 page_cnt, page_size;
184 struct qlnxr_srq *srq;
187 dev = get_qlnxr_dev((ibpd->device));
190 QL_DPRINT12(ha, "enter\n");
192 ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
194 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
196 QL_DPRINT11(ha, "cannot allocate memory for srq\n");
197 return NULL; //@@@ : TODO what to return here?
201 hw_srq = &srq->hw_srq;
202 spin_lock_init(&srq->lock);
203 memset(&in_params, 0, sizeof(in_params));
205 if (udata && ibpd->uobject && ibpd->uobject->context) {
206 ib_ctx = ibpd->uobject->context;
207 ctx = get_qlnxr_ucontext(ib_ctx);
209 memset(&ureq, 0, sizeof(ureq));
210 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
212 QL_DPRINT11(ha, "problem"
213 " copying data from user space\n");
217 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
221 page_cnt = srq->usrq.pbl_info.num_pbes;
222 pbl_base_addr = srq->usrq.pbl_tbl->pa;
223 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
224 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT
225 // page_size = BIT(srq->usrq.umem->page_shift);
227 page_size = srq->usrq.umem->page_size;
229 struct ecore_chain *pbl;
230 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr);
235 page_cnt = ecore_chain_get_page_cnt(pbl);
236 pbl_base_addr = ecore_chain_get_pbl_phys(pbl);
237 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
238 page_size = pbl->elem_per_page << 4;
241 in_params.pd_id = pd->pd_id;
242 in_params.pbl_base_addr = pbl_base_addr;
243 in_params.prod_pair_addr = phy_prod_pair_addr;
244 in_params.num_pages = page_cnt;
245 in_params.page_size = page_size;
247 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
251 srq->srq_id = out_params.srq_id;
254 ret = qlnxr_copy_srq_uresp(dev, srq, udata);
259 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
262 memset(&in_params, 0, sizeof(in_params));
263 destroy_in_params.srq_id = srq->srq_id;
264 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
268 qlnxr_free_srq_user_params(srq);
270 qlnxr_free_srq_kernel_params(srq);
274 return ERR_PTR(-EFAULT);
278 qlnxr_destroy_srq(struct ib_srq *ibsrq)
280 struct qlnxr_dev *dev;
281 struct qlnxr_srq *srq;
283 struct ecore_rdma_destroy_srq_in_params in_params;
285 srq = get_qlnxr_srq(ibsrq);
289 memset(&in_params, 0, sizeof(in_params));
290 in_params.srq_id = srq->srq_id;
292 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);
294 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)
295 qlnxr_free_srq_user_params(srq);
297 qlnxr_free_srq_kernel_params(srq);
299 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
305 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
306 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
308 struct qlnxr_dev *dev;
309 struct qlnxr_srq *srq;
311 struct ecore_rdma_modify_srq_in_params in_params;
314 srq = get_qlnxr_srq(ibsrq);
318 QL_DPRINT12(ha, "enter\n");
319 if (attr_mask & IB_SRQ_MAX_WR) {
320 QL_DPRINT12(ha, "invalid attribute mask=0x%x"
321 " specified for %p\n", attr_mask, srq);
325 if (attr_mask & IB_SRQ_LIMIT) {
326 if (attr->srq_limit >= srq->hw_srq.max_wr) {
327 QL_DPRINT12(ha, "invalid srq_limit=0x%x"
328 " (max_srq_limit = 0x%x)\n",
329 attr->srq_limit, srq->hw_srq.max_wr);
332 memset(&in_params, 0, sizeof(in_params));
333 in_params.srq_id = srq->srq_id;
334 in_params.wqe_limit = attr->srq_limit;
335 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);
340 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);
345 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
347 struct qlnxr_dev *dev;
348 struct qlnxr_srq *srq;
350 struct ecore_rdma_device *qattr;
351 srq = get_qlnxr_srq(ibsrq);
354 //qattr = &dev->attr;
355 qattr = ecore_rdma_query_device(dev->rdma_ctx);
356 QL_DPRINT12(ha, "enter\n");
358 if (!dev->rdma_ctx) {
359 QL_DPRINT12(ha, "called with invalid params"
360 " rdma_ctx is NULL\n");
364 srq_attr->srq_limit = qattr->max_srq;
365 srq_attr->max_wr = qattr->max_srq_wr;
366 srq_attr->max_sge = qattr->max_sge;
368 QL_DPRINT12(ha, "exit\n");
372 /* Increment srq wr producer by one */
374 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info)
379 /* Increment srq wr consumer by one */
381 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info)
386 /* get_port_immutable verb is not available in FreeBSD */
389 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
390 struct ib_port_immutable *immutable)
392 struct qlnxr_dev *dev;
394 dev = get_qlnxr_dev(ibdev);
397 QL_DPRINT12(ha, "entered but not implemented!!!\n");
402 qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
403 struct ib_recv_wr **bad_wr)
405 struct qlnxr_dev *dev;
406 struct qlnxr_srq *srq;
408 struct qlnxr_srq_hwq_info *hw_srq;
409 struct ecore_chain *pbl;
414 srq = get_qlnxr_srq(ibsrq);
417 hw_srq = &srq->hw_srq;
419 QL_DPRINT12(ha, "enter\n");
420 spin_lock_irqsave(&srq->lock, flags);
422 pbl = &srq->hw_srq.pbl;
424 struct rdma_srq_wqe_header *hdr;
427 if (!qlnxr_srq_elem_left(hw_srq) ||
428 wr->num_sge > srq->hw_srq.max_sges) {
429 QL_DPRINT11(ha, "WR cannot be posted"
430 " (%d, %d) || (%d > %d)\n",
431 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
432 wr->num_sge, srq->hw_srq.max_sges);
438 hdr = ecore_chain_produce(pbl);
439 num_sge = wr->num_sge;
440 /* Set number of sge and WR id in header */
441 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
443 /* PBL is maintained in case of WR granularity.
444 * So increment WR producer in case we post a WR.
446 qlnxr_inc_srq_wr_prod(hw_srq);
450 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n",
451 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
453 for (i = 0; i < wr->num_sge; i++) {
454 struct rdma_srq_sge *srq_sge =
455 ecore_chain_produce(pbl);
456 /* Set SGE length, lkey and address */
457 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
458 wr->sg_list[i].length, wr->sg_list[i].lkey);
460 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",
461 i, srq_sge->length, srq_sge->l_key,
462 srq_sge->addr.hi, srq_sge->addr.lo);
467 * SRQ prod is 8 bytes. Need to update SGE prod in index
468 * in first 4 bytes and need to update WQE prod in next
471 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;
472 offset = offsetof(struct rdma_srq_producers, wqe_prod);
473 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
475 /* Flush prod after updating it */
480 QL_DPRINT12(ha, "Elements in SRQ: %d\n",
481 ecore_chain_get_elem_left(pbl));
483 spin_unlock_irqrestore(&srq->lock, flags);
484 QL_DPRINT12(ha, "exit\n");
489 #if __FreeBSD_version < 1102000
490 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
492 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
493 struct ib_udata *udata)
494 #endif /* #if __FreeBSD_version < 1102000 */
497 struct qlnxr_dev *dev;
498 struct ecore_rdma_device *qattr;
501 dev = get_qlnxr_dev(ibdev);
504 QL_DPRINT12(ha, "enter\n");
506 #if __FreeBSD_version > 1102000
507 if (udata->inlen || udata->outlen)
509 #endif /* #if __FreeBSD_version > 1102000 */
511 if (dev->rdma_ctx == NULL) {
515 qattr = ecore_rdma_query_device(dev->rdma_ctx);
517 memset(attr, 0, sizeof *attr);
519 attr->fw_ver = qattr->fw_ver;
520 attr->sys_image_guid = qattr->sys_image_guid;
521 attr->max_mr_size = qattr->max_mr_size;
522 attr->page_size_cap = qattr->page_size_caps;
523 attr->vendor_id = qattr->vendor_id;
524 attr->vendor_part_id = qattr->vendor_part_id;
525 attr->hw_ver = qattr->hw_ver;
526 attr->max_qp = qattr->max_qp;
527 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
528 IB_DEVICE_RC_RNR_NAK_GEN |
529 IB_DEVICE_LOCAL_DMA_LKEY |
530 IB_DEVICE_MEM_MGT_EXTENSIONS;
532 attr->max_sge = qattr->max_sge;
533 attr->max_sge_rd = qattr->max_sge;
534 attr->max_cq = qattr->max_cq;
535 attr->max_cqe = qattr->max_cqe;
536 attr->max_mr = qattr->max_mr;
537 attr->max_mw = qattr->max_mw;
538 attr->max_pd = qattr->max_pd;
539 attr->atomic_cap = dev->atomic_cap;
540 attr->max_fmr = qattr->max_fmr;
541 attr->max_map_per_fmr = 16; /* TBD: FMR */
543 /* There is an implicit assumption in some of the ib_xxx apps that the
544 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in
545 * communication the qp_rd_atom is passed to the other side and used as
546 * init_rd_atom without check device capabilities for init_rd_atom.
547 * for this reason, we set the qp_rd_atom to be the minimum between the
548 * two...There is an additional assumption in mlx4 driver that the
549 * values are power of two, fls is performed on the value - 1, which
550 * in fact gives a larger power of two for values which are not a power
551 * of two. This should be fixed in mlx4 driver, but until then ->
552 * we provide a value that is a power of two in our code.
554 attr->max_qp_init_rd_atom =
555 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
556 attr->max_qp_rd_atom =
557 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
558 attr->max_qp_init_rd_atom);
560 attr->max_srq = qattr->max_srq;
561 attr->max_srq_sge = qattr->max_srq_sge;
562 attr->max_srq_wr = qattr->max_srq_wr;
564 /* TODO: R&D to more properly configure the following */
565 attr->local_ca_ack_delay = qattr->dev_ack_delay;
566 attr->max_fast_reg_page_list_len = qattr->max_mr/8;
567 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;
568 attr->max_ah = qattr->max_ah;
570 QL_DPRINT12(ha, "exit\n");
575 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width)
579 *ib_speed = IB_SPEED_SDR;
580 *ib_width = IB_WIDTH_1X;
583 *ib_speed = IB_SPEED_QDR;
584 *ib_width = IB_WIDTH_1X;
588 *ib_speed = IB_SPEED_DDR;
589 *ib_width = IB_WIDTH_4X;
593 *ib_speed = IB_SPEED_EDR;
594 *ib_width = IB_WIDTH_1X;
598 *ib_speed = IB_SPEED_QDR;
599 *ib_width = IB_WIDTH_4X;
603 *ib_speed = IB_SPEED_QDR;
604 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50...
608 *ib_speed = IB_SPEED_EDR;
609 *ib_width = IB_WIDTH_4X;
614 *ib_speed = IB_SPEED_SDR;
615 *ib_width = IB_WIDTH_1X;
621 qlnxr_query_port(struct ib_device *ibdev, uint8_t port,
622 struct ib_port_attr *attr)
624 struct qlnxr_dev *dev;
625 struct ecore_rdma_port *rdma_port;
628 dev = get_qlnxr_dev(ibdev);
631 QL_DPRINT12(ha, "enter\n");
634 QL_DPRINT12(ha, "port [%d] > 1 \n", port);
638 if (dev->rdma_ctx == NULL) {
639 QL_DPRINT12(ha, "rdma_ctx == NULL\n");
643 rdma_port = ecore_rdma_query_port(dev->rdma_ctx);
644 memset(attr, 0, sizeof *attr);
646 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {
647 attr->state = IB_PORT_ACTIVE;
648 attr->phys_state = 5;
650 attr->state = IB_PORT_DOWN;
651 attr->phys_state = 3;
654 attr->max_mtu = IB_MTU_4096;
655 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu);
660 attr->port_cap_flags = 0;
662 if (QLNX_IS_IWARP(dev)) {
663 attr->gid_tbl_len = 1;
664 attr->pkey_tbl_len = 1;
666 attr->gid_tbl_len = QLNXR_MAX_SGID;
667 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;
670 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
671 attr->qkey_viol_cntr = 0;
673 get_link_speed_and_width(rdma_port->link_speed,
674 &attr->active_speed, &attr->active_width);
676 attr->max_msg_sz = rdma_port->max_msg_size;
677 attr->max_vl_num = 4; /* TODO -> figure this one out... */
679 QL_DPRINT12(ha, "state = %d phys_state = %d "
680 " link_speed = %d active_speed = %d active_width = %d"
681 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"
682 " max_msg_sz = 0x%x max_vl_num = 0x%x \n",
683 attr->state, attr->phys_state,
684 rdma_port->link_speed, attr->active_speed,
685 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,
686 attr->max_msg_sz, attr->max_vl_num);
688 QL_DPRINT12(ha, "exit\n");
693 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask,
694 struct ib_port_modify *props)
696 struct qlnxr_dev *dev;
699 dev = get_qlnxr_dev(ibdev);
702 QL_DPRINT12(ha, "enter\n");
705 QL_DPRINT12(ha, "port (%d) > 1\n", port);
709 QL_DPRINT12(ha, "exit\n");
714 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
716 struct qlnxr_dev *dev;
719 dev = get_qlnxr_dev(ibdev);
722 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);
724 return IB_LINK_LAYER_ETHERNET;
728 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
729 struct ib_udata *udata)
731 struct qlnxr_pd *pd = NULL;
734 struct qlnxr_dev *dev;
737 dev = get_qlnxr_dev(ibdev);
740 QL_DPRINT12(ha, "ibdev = %p context = %p"
741 " udata = %p enter\n", ibdev, context, udata);
743 if (dev->rdma_ctx == NULL) {
744 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
749 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
752 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
756 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
758 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
764 if (udata && context) {
766 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
768 QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
769 ecore_rdma_free_pd(dev->rdma_ctx, pd_id);
773 pd->uctx = get_qlnxr_ucontext(context);
777 atomic_add_rel_32(&dev->pd_count, 1);
778 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
779 pd, pd_id, dev->pd_count);
785 QL_DPRINT12(ha, "exit -1\n");
790 qlnxr_dealloc_pd(struct ib_pd *ibpd)
793 struct qlnxr_dev *dev;
796 pd = get_qlnxr_pd(ibpd);
797 dev = get_qlnxr_dev((ibpd->device));
800 QL_DPRINT12(ha, "enter\n");
803 QL_DPRINT11(ha, "pd = NULL\n");
805 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
807 atomic_subtract_rel_32(&dev->pd_count, 1);
808 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
809 pd, pd->pd_id, dev->pd_count);
812 QL_DPRINT12(ha, "exit\n");
816 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge)
817 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */
818 /* Should be part of HSI */
819 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */
820 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
822 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *);
823 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *);
826 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
828 struct qlnxr_dev *dev;
831 dev = get_qlnxr_dev(ibdev);
834 QL_DPRINT12(ha, "enter index = 0x%x\n", index);
836 if (index > QLNXR_ROCE_PKEY_TABLE_LEN)
839 *pkey = QLNXR_ROCE_PKEY_DEFAULT;
841 QL_DPRINT12(ha, "exit\n");
847 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask,
852 QL_DPRINT12(ha, "enter \n");
856 #if __FreeBSD_version >= 1100000
859 #if __FreeBSD_version >= 1102000
862 dgid = &attr->ah_attr.grh.dgid;
863 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
865 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) {
866 *vlan_id = tmp_vlan_id;
870 tmp_vlan_id = attr->vlan_id;
872 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) {
873 *vlan_id = tmp_vlan_id;
877 #endif /* #if __FreeBSD_version > 1102000 */
882 #endif /* #if __FreeBSD_version >= 1100000 */
884 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);
890 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr,
892 struct qlnxr_dev *dev,
894 struct ecore_rdma_modify_qp_in_params *qp_params)
901 QL_DPRINT12(ha, "enter\n");
903 memcpy(&qp_params->sgid.bytes[0],
904 &dev->sgid_tbl[qp->sgid_idx].raw[0],
905 sizeof(qp_params->sgid.bytes));
906 memcpy(&qp_params->dgid.bytes[0],
907 &attr->ah_attr.grh.dgid.raw[0],
908 sizeof(qp_params->dgid));
910 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);
912 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {
913 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
914 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
917 QL_DPRINT12(ha, "exit\n");
924 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
931 QL_DPRINT12(ha, "enter\n");
933 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
935 QL_DPRINT11(ha, "mm = NULL\n");
939 mm->key.phy_addr = phy_addr;
941 /* This function might be called with a length which is not a multiple
942 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
943 * forces this granularity by increasing the requested size if needed.
944 * When qedr_mmap is called, it will search the list with the updated
945 * length as a key. To prevent search failures, the length is rounded up
946 * in advance to PAGE_SIZE.
948 mm->key.len = roundup(len, PAGE_SIZE);
949 INIT_LIST_HEAD(&mm->entry);
951 mutex_lock(&uctx->mm_list_lock);
952 list_add(&mm->entry, &uctx->mm_head);
953 mutex_unlock(&uctx->mm_list_lock);
955 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
956 (unsigned long long)mm->key.phy_addr,
957 (unsigned long)mm->key.len, uctx);
963 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
971 QL_DPRINT12(ha, "enter\n");
973 mutex_lock(&uctx->mm_list_lock);
974 list_for_each_entry(mm, &uctx->mm_head, entry) {
975 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
981 mutex_unlock(&uctx->mm_list_lock);
984 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",
985 mm->key.phy_addr, mm->key.len, uctx, found);
991 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
992 struct ib_udata *udata)
995 struct qlnxr_ucontext *ctx;
996 struct qlnxr_alloc_ucontext_resp uresp;
997 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
998 qlnx_host_t *ha = dev->ha;
999 struct ecore_rdma_add_user_out_params oparams;
1002 return ERR_PTR(-EFAULT);
1005 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1007 return ERR_PTR(-ENOMEM);
1009 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
1012 "Failed to allocate a DPI for a new RoCE application "
1013 ",rc = %d. To overcome this, consider to increase "
1014 "the number of DPIs, increase the doorbell BAR size "
1015 "or just close unnecessary RoCE applications. In "
1016 "order to increase the number of DPIs consult the "
1021 ctx->dpi = oparams.dpi;
1022 ctx->dpi_addr = oparams.dpi_addr;
1023 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
1024 ctx->dpi_size = oparams.dpi_size;
1025 INIT_LIST_HEAD(&ctx->mm_head);
1026 mutex_init(&ctx->mm_list_lock);
1028 memset(&uresp, 0, sizeof(uresp));
1029 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled)
1030 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out
1031 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled)
1032 < udata->outlen ? 1 : 0; //TODO: figure this out
1033 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count)
1034 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out
1035 uresp.db_pa = ctx->dpi_phys_addr;
1036 uresp.db_size = ctx->dpi_size;
1037 uresp.max_send_wr = dev->attr.max_sqe;
1038 uresp.max_recv_wr = dev->attr.max_rqe;
1039 uresp.max_srq_wr = dev->attr.max_srq_wr;
1040 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
1041 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
1042 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
1043 uresp.max_cqes = QLNXR_MAX_CQES;
1045 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1051 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
1054 QL_DPRINT12(ha, "Allocated user context %p\n",
1057 return &ctx->ibucontext;
1064 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
1066 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
1067 struct qlnxr_dev *dev = uctx->dev;
1068 qlnx_host_t *ha = dev->ha;
1069 struct qlnxr_mm *mm, *tmp;
1072 QL_DPRINT12(ha, "Deallocating user context %p\n",
1076 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
1079 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
1080 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"
1082 mm->key.phy_addr, mm->key.len, uctx);
1083 list_del(&mm->entry);
1091 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
1093 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context);
1094 struct qlnxr_dev *dev = get_qlnxr_dev((context->device));
1095 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
1097 unsigned long len = (vma->vm_end - vma->vm_start);
1104 #if __FreeBSD_version > 1102000
1105 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
1107 unmapped_db = dev->db_phys_addr;
1108 #endif /* #if __FreeBSD_version > 1102000 */
1110 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"
1111 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
1112 vm_page, vma->vm_pgoff, unmapped_db,
1115 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
1116 QL_DPRINT11(ha, "Vma_start not page aligned "
1117 "vm_start = %ld vma_end = %ld\n", vma->vm_start,
1122 found = qlnxr_search_mmap(ucontext, vm_page, len);
1124 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n",
1129 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1131 #if __FreeBSD_version > 1102000
1133 if ((vm_page < unmapped_db) ||
1134 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {
1135 QL_DPRINT11(ha, "failed pages are outside of dpi;"
1136 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",
1137 vm_page, unmapped_db, ucontext->dpi_size);
1141 if (vma->vm_flags & VM_READ) {
1142 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n");
1146 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1147 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
1152 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
1155 QL_DPRINT12(ha, "Mapping doorbell bar\n");
1157 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1159 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1160 PAGE_SIZE, vma->vm_page_prot);
1162 QL_DPRINT12(ha, "Mapping chains\n");
1163 rc = io_remap_pfn_range(vma, vma->vm_start,
1164 vma->vm_pgoff, len, vma->vm_page_prot);
1167 #endif /* #if __FreeBSD_version > 1102000 */
1169 QL_DPRINT12(ha, "exit [%d]\n", rc);
1174 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc)
1176 struct qlnxr_mr *mr;
1177 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1178 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
1184 QL_DPRINT12(ha, "enter\n");
1186 if (acc & IB_ACCESS_MW_BIND) {
1187 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n");
1190 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1193 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);
1197 mr->type = QLNXR_MR_DMA;
1199 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1201 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);
1205 /* index only, 18 bit long, lkey = itid << 8 | key */
1206 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1207 mr->hw_mr.pd = pd->pd_id;
1208 mr->hw_mr.local_read = 1;
1209 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1210 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1211 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1212 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1213 mr->hw_mr.dma_mr = true;
1215 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1217 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);
1221 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1223 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1224 mr->hw_mr.remote_atomic) {
1225 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1228 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);
1233 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1237 QL_DPRINT12(ha, "exit [%d]\n", rc);
1243 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info,
1244 struct qlnxr_pbl *pbl)
1251 QL_DPRINT12(ha, "enter\n");
1253 for (i = 0; i < pbl_info->num_pbls; i++) {
1256 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,
1257 pbl_info->pbl_size);
1261 QL_DPRINT12(ha, "exit\n");
1265 #define MIN_FW_PBL_PAGE_SIZE (4*1024)
1266 #define MAX_FW_PBL_PAGE_SIZE (64*1024)
1268 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
1269 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
1270 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE)
1272 static struct qlnxr_pbl *
1273 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev,
1274 struct qlnxr_pbl_info *pbl_info, gfp_t flags)
1278 dma_addr_t *pbl_main_tbl;
1279 struct qlnxr_pbl *pbl_table;
1285 QL_DPRINT12(ha, "enter\n");
1287 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);
1290 QL_DPRINT12(ha, "pbl_table = NULL\n");
1294 for (i = 0; i < pbl_info->num_pbls; i++) {
1295 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);
1297 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i);
1301 memset(va, 0, pbl_info->pbl_size);
1302 pbl_table[i].va = va;
1303 pbl_table[i].pa = pa;
1306 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
1307 * the first one with physical pointers to all of the rest
1309 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
1310 for (i = 0; i < pbl_info->num_pbls - 1; i++)
1311 pbl_main_tbl[i] = pbl_table[i + 1].pa;
1313 QL_DPRINT12(ha, "exit\n");
1317 qlnxr_free_pbl(dev, pbl_info, pbl_table);
1319 QL_DPRINT12(ha, "exit with error\n");
1324 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev,
1325 struct qlnxr_pbl_info *pbl_info,
1327 int two_layer_capable)
1336 QL_DPRINT12(ha, "enter\n");
1338 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
1339 if (num_pbes > MAX_PBES_TWO_LAYER) {
1340 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n",
1345 /* calculate required pbl page size */
1346 pbl_size = MIN_FW_PBL_PAGE_SIZE;
1347 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
1348 NUM_PBES_ON_PAGE(pbl_size);
1350 while (pbl_capacity < num_pbes) {
1352 pbl_capacity = pbl_size / sizeof(u64);
1353 pbl_capacity = pbl_capacity * pbl_capacity;
1356 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
1357 num_pbls++; /* One for the layer0 ( points to the pbls) */
1358 pbl_info->two_layered = true;
1360 /* One layered PBL */
1362 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \
1363 roundup_pow_of_two((num_pbes * sizeof(u64))));
1364 pbl_info->two_layered = false;
1367 pbl_info->num_pbls = num_pbls;
1368 pbl_info->pbl_size = pbl_size;
1369 pbl_info->num_pbes = num_pbes;
1371 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n",
1372 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
1377 #define upper_32_bits(x) (uint32_t)(x >> 32)
1378 #define lower_32_bits(x) (uint32_t)(x)
1381 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem,
1382 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
1384 struct regpair *pbe;
1385 struct qlnxr_pbl *pbl_tbl;
1386 struct scatterlist *sg;
1387 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
1390 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1392 struct ib_umem_chunk *chunk = NULL;
1400 QL_DPRINT12(ha, "enter\n");
1403 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
1407 if (!pbl_info->num_pbes) {
1408 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
1412 /* If we have a two layered pbl, the first pbl points to the rest
1413 * of the pbls and the first entry lays on the second pbl in the table
1415 if (pbl_info->two_layered)
1420 pbe = (struct regpair *)pbl_tbl->va;
1422 QL_DPRINT12(ha, "pbe is NULL\n");
1428 shift = ilog2(umem->page_size);
1430 #ifndef DEFINE_IB_UMEM_WITH_CHUNK
1432 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1435 list_for_each_entry(chunk, &umem->chunk_list, list) {
1436 /* get all the dma regions from the chunk. */
1437 for (i = 0; i < chunk->nmap; i++) {
1438 sg = &chunk->page_list[i];
1440 pages = sg_dma_len(sg) >> shift;
1441 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
1442 /* store the page address in pbe */
1444 cpu_to_le32(sg_dma_address(sg) +
1445 (umem->page_size * pg_cnt));
1447 cpu_to_le32(upper_32_bits
1448 ((sg_dma_address(sg) +
1449 umem->page_size * pg_cnt)));
1452 "Populate pbl table:"
1453 " pbe->addr=0x%x:0x%x "
1454 " pbe_cnt = %d total_num_pbes=%d"
1455 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
1456 total_num_pbes, pbe);
1462 if (total_num_pbes == pbl_info->num_pbes)
1465 /* if the given pbl is full storing the pbes,
1469 (pbl_info->pbl_size / sizeof(u64))) {
1471 pbe = (struct regpair *)pbl_tbl->va;
1475 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1479 QL_DPRINT12(ha, "exit\n");
1484 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info)
1486 struct qlnxr_pbl *pbl, *tmp;
1491 QL_DPRINT12(ha, "enter\n");
1493 if (info->pbl_table)
1494 list_add_tail(&info->pbl_table->list_entry,
1495 &info->free_pbl_list);
1497 if (!list_empty(&info->inuse_pbl_list))
1498 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
1500 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
1501 list_del(&pbl->list_entry);
1502 qlnxr_free_pbl(dev, &info->pbl_info, pbl);
1504 QL_DPRINT12(ha, "exit\n");
1510 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info,
1511 size_t page_list_len, bool two_layered)
1514 struct qlnxr_pbl *tmp;
1519 QL_DPRINT12(ha, "enter\n");
1521 INIT_LIST_HEAD(&info->free_pbl_list);
1522 INIT_LIST_HEAD(&info->inuse_pbl_list);
1524 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,
1525 page_list_len, two_layered);
1527 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);
1531 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1533 if (!info->pbl_table) {
1535 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n");
1539 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);
1541 /* in usual case we use 2 PBLs, so we add one to free
1542 * list and allocating another one
1544 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1547 QL_DPRINT11(ha, "Extra PBL is not allocated\n");
1548 goto done; /* it's OK if second allocation fails, so rc = 0*/
1551 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
1553 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);
1557 free_mr_info(dev, info);
1559 QL_DPRINT12(ha, "exit [%d]\n", rc);
1566 #if __FreeBSD_version >= 1102000
1567 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1568 u64 usr_addr, int acc, struct ib_udata *udata)
1570 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1571 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id)
1572 #endif /* #if __FreeBSD_version >= 1102000 */
1575 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1576 struct qlnxr_mr *mr;
1577 struct qlnxr_pd *pd;
1582 QL_DPRINT12(ha, "enter\n");
1584 pd = get_qlnxr_pd(ibpd);
1586 QL_DPRINT12(ha, "qedr_register user mr pd = %d"
1587 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
1588 pd->pd_id, start, len, usr_addr, acc);
1590 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
1592 "(acc & IB_ACCESS_REMOTE_WRITE &&"
1593 " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
1594 return ERR_PTR(-EINVAL);
1597 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1599 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
1603 mr->type = QLNXR_MR_USER;
1605 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
1606 if (IS_ERR(mr->umem)) {
1608 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);
1612 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
1615 "qlnxr_init_mr_info failed [%d]\n", rc);
1619 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
1620 &mr->info.pbl_info);
1622 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1625 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
1629 /* index only, 18 bit long, lkey = itid << 8 | key */
1630 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1632 mr->hw_mr.pd = pd->pd_id;
1633 mr->hw_mr.local_read = 1;
1634 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1635 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1636 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1637 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1638 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
1639 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
1640 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
1641 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
1642 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */
1644 #if __FreeBSD_version >= 1102000
1645 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
1647 mr->hw_mr.fbo = mr->umem->offset;
1649 mr->hw_mr.length = len;
1650 mr->hw_mr.vaddr = usr_addr;
1651 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
1652 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
1653 mr->hw_mr.dma_mr = false;
1655 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1657 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
1661 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1662 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1663 mr->hw_mr.remote_atomic)
1664 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1666 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);
1671 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1673 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1677 QL_DPRINT12(ha, "exit [%d]\n", rc);
1678 return (ERR_PTR(rc));
1682 qlnxr_dereg_mr(struct ib_mr *ib_mr)
1684 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr);
1685 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
1691 QL_DPRINT12(ha, "enter\n");
1693 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))
1694 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1696 /* it could be user registered memory. */
1698 ib_umem_release(mr->umem);
1704 QL_DPRINT12(ha, "exit\n");
1709 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev,
1710 struct qlnxr_cq *cq, struct ib_udata *udata)
1712 struct qlnxr_create_cq_uresp uresp;
1718 QL_DPRINT12(ha, "enter\n");
1720 memset(&uresp, 0, sizeof(uresp));
1722 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1723 uresp.icid = cq->icid;
1725 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1728 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",
1732 QL_DPRINT12(ha, "exit [%d]\n", rc);
1737 consume_cqe(struct qlnxr_cq *cq)
1740 if (cq->latest_cqe == cq->toggle_cqe)
1741 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1743 cq->latest_cqe = ecore_chain_consume(&cq->pbl);
1747 qlnxr_align_cq_entries(int entries)
1749 u64 size, aligned_size;
1751 /* We allocate an extra entry that we don't report to the FW.
1753 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes
1754 * (for performance purposes). Allocating an extra entry and telling
1755 * the FW we have less prevents overwriting the first entry in case of
1756 * a wrap i.e. when the FW writes the last entry and the application
1757 * hasn't read the first one.
1759 size = (entries + 1) * QLNXR_CQE_SIZE;
1761 /* We align to PAGE_SIZE.
1763 * Since the CQ is going to be mapped and the mapping is anyhow in whole
1764 * kernel pages we benefit from the possibly extra CQEs.
1766 aligned_size = ALIGN(size, PAGE_SIZE);
1768 /* note: for CQs created in user space the result of this function
1769 * should match the size mapped in user space
1771 return (aligned_size / QLNXR_CQE_SIZE);
1775 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev,
1776 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len,
1777 int access, int dmasync, int alloc_and_init)
1785 QL_DPRINT12(ha, "enter\n");
1787 q->buf_addr = buf_addr;
1788 q->buf_len = buf_len;
1790 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"
1791 " dmasync : %x\n", q->buf_addr, q->buf_len,
1794 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
1796 if (IS_ERR(q->umem)) {
1797 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));
1798 return PTR_ERR(q->umem);
1801 page_cnt = ib_umem_page_count(q->umem);
1802 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,
1803 0 /* SQ and RQ don't support dual layer pbl.
1804 * CQ may, but this is yet uncoded.
1807 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);
1811 if (alloc_and_init) {
1812 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
1815 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1820 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
1822 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
1825 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1831 QL_DPRINT12(ha, "exit\n");
1835 ib_umem_release(q->umem);
1838 QL_DPRINT12(ha, "exit [%d]\n", rc);
1842 #if __FreeBSD_version >= 1102000
1845 qlnxr_create_cq(struct ib_device *ibdev,
1846 const struct ib_cq_init_attr *attr,
1847 struct ib_ucontext *ib_ctx,
1848 struct ib_udata *udata)
1852 #if __FreeBSD_version >= 1100000
1855 qlnxr_create_cq(struct ib_device *ibdev,
1856 struct ib_cq_init_attr *attr,
1857 struct ib_ucontext *ib_ctx,
1858 struct ib_udata *udata)
1863 qlnxr_create_cq(struct ib_device *ibdev,
1866 struct ib_ucontext *ib_ctx,
1867 struct ib_udata *udata)
1868 #endif /* #if __FreeBSD_version >= 1100000 */
1870 #endif /* #if __FreeBSD_version >= 1102000 */
1872 struct qlnxr_ucontext *ctx;
1873 struct ecore_rdma_destroy_cq_out_params destroy_oparams;
1874 struct ecore_rdma_destroy_cq_in_params destroy_iparams;
1875 struct qlnxr_dev *dev;
1876 struct ecore_rdma_create_cq_in_params params;
1877 struct qlnxr_create_cq_ureq ureq;
1879 #if __FreeBSD_version >= 1100000
1880 int vector = attr->comp_vector;
1881 int entries = attr->cqe;
1883 struct qlnxr_cq *cq;
1884 int chain_entries, rc, page_cnt;
1889 dev = get_qlnxr_dev(ibdev);
1892 QL_DPRINT12(ha, "called from %s. entries = %d, "
1894 (udata ? "User Lib" : "Kernel"), entries, vector);
1896 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params));
1897 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params));
1898 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params));
1900 if (entries > QLNXR_MAX_CQES) {
1902 "the number of entries %d is too high. "
1903 "Must be equal or below %d.\n",
1904 entries, QLNXR_MAX_CQES);
1905 return ERR_PTR(-EINVAL);
1907 chain_entries = qlnxr_align_cq_entries(entries);
1908 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
1910 cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
1913 return ERR_PTR(-ENOMEM);
1916 memset(&ureq, 0, sizeof(ureq));
1918 if (ib_copy_from_udata(&ureq, udata,
1919 min(sizeof(ureq), udata->inlen))) {
1920 QL_DPRINT11(ha, "ib_copy_from_udata failed\n");
1925 QL_DPRINT11(ha, "ureq.len == 0\n");
1929 cq->cq_type = QLNXR_CQ_TYPE_USER;
1931 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
1932 IB_ACCESS_LOCAL_WRITE, 1, 1);
1934 pbl_ptr = cq->q.pbl_tbl->pa;
1935 page_cnt = cq->q.pbl_info.num_pbes;
1936 cq->ibcq.cqe = chain_entries;
1938 cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
1940 rc = ecore_chain_alloc(&dev->ha->cdev,
1941 ECORE_CHAIN_USE_TO_CONSUME,
1942 ECORE_CHAIN_MODE_PBL,
1943 ECORE_CHAIN_CNT_TYPE_U32,
1945 sizeof(union roce_cqe),
1951 page_cnt = ecore_chain_get_page_cnt(&cq->pbl);
1952 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);
1953 cq->ibcq.cqe = cq->pbl.capacity;
1956 params.cq_handle_hi = upper_32_bits((uintptr_t)cq);
1957 params.cq_handle_lo = lower_32_bits((uintptr_t)cq);
1958 params.cnq_id = vector;
1959 params.cq_size = chain_entries - 1;
1960 params.pbl_num_pages = page_cnt;
1961 params.pbl_ptr = pbl_ptr;
1962 params.pbl_two_level = 0;
1964 if (ib_ctx != NULL) {
1965 ctx = get_qlnxr_ucontext(ib_ctx);
1966 params.dpi = ctx->dpi;
1968 params.dpi = dev->dpi;
1971 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
1976 cq->sig = QLNXR_CQ_MAGIC_NUMBER;
1977 spin_lock_init(&cq->cq_lock);
1980 rc = qlnxr_copy_cq_uresp(dev, cq, udata);
1984 /* Generate doorbell address.
1985 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.
1986 * TODO: consider moving to device scope as it is a function of
1988 * TODO: add ifdef if plan to support 16 bit.
1990 cq->db_addr = dev->db_addr +
1991 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1992 cq->db.data.icid = cq->icid;
1993 cq->db.data.params = DB_AGG_CMD_SET <<
1994 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1996 /* point to the very last element, passing it we will toggle */
1997 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);
1998 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
2000 /* must be different from pbl_toggle */
2001 cq->latest_cqe = NULL;
2003 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
2006 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"
2007 " number of entries = 0x%x\n",
2008 cq->icid, cq, params.cq_size);
2009 QL_DPRINT12(ha,"cq_addr = %p\n", cq);
2013 destroy_iparams.icid = cq->icid;
2014 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);
2017 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2019 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2022 ib_umem_release(cq->q.umem);
2026 QL_DPRINT12(ha, "exit error\n");
2028 return ERR_PTR(-EINVAL);
2031 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
2034 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2039 QL_DPRINT12(ha, "enter/exit\n");
2045 qlnxr_destroy_cq(struct ib_cq *ibcq)
2047 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
2048 struct ecore_rdma_destroy_cq_out_params oparams;
2049 struct ecore_rdma_destroy_cq_in_params iparams;
2050 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
2056 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);
2060 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation
2061 * that all completions with notification are dealt with. The rest
2062 * of the completions are not interesting
2065 /* GSIs CQs are handled by driver, so they don't exist in the FW */
2067 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {
2069 iparams.icid = cq->icid;
2071 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
2074 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
2079 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
2080 ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2083 if (ibcq->uobject && ibcq->uobject->context) {
2084 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2085 ib_umem_release(cq->q.umem);
2092 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
2098 qlnxr_check_qp_attrs(struct ib_pd *ibpd,
2099 struct qlnxr_dev *dev,
2100 struct ib_qp_init_attr *attrs,
2101 struct ib_udata *udata)
2103 struct ecore_rdma_device *qattr;
2106 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2109 QL_DPRINT12(ha, "enter\n");
2111 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);
2112 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);
2113 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);
2115 #if __FreeBSD_version < 1102000
2116 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type);
2119 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);
2120 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);
2121 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);
2122 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);
2123 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);
2124 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",
2125 attrs->cap.max_inline_data);
2127 #if __FreeBSD_version < 1102000
2128 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n",
2129 attrs->cap.qpg_tss_mask_sz);
2132 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);
2133 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);
2134 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);
2135 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);
2136 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);
2137 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",
2138 (void *)qattr->sys_image_guid);
2139 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);
2140 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);
2141 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);
2142 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);
2143 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);
2144 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);
2145 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",
2146 qattr->max_qp_resp_rd_atomic_resc);
2147 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",
2148 qattr->max_qp_req_rd_atomic_resc);
2149 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",
2150 qattr->max_dev_resp_rd_atomic_resc);
2151 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);
2152 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);
2153 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);
2154 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);
2155 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);
2156 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);
2157 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);
2158 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);
2159 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",
2160 qattr->max_mr_mw_fmr_pbl);
2161 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",
2162 (void *)qattr->max_mr_mw_fmr_size);
2163 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);
2164 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);
2165 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);
2166 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);
2167 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",
2168 qattr->max_stats_queues);
2169 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);
2170 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",
2171 (void *)qattr->page_size_caps);
2172 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);
2173 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);
2174 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",
2175 qattr->bad_pkey_counter);
2177 if ((attrs->qp_type == IB_QPT_GSI) && udata) {
2178 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n");
2182 if (udata && !(ibpd->uobject && ibpd->uobject->context)) {
2183 QL_DPRINT12(ha, "called from user without context\n");
2187 /* QP0... attrs->qp_type == IB_QPT_GSI */
2188 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
2189 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n",
2193 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {
2194 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n");
2197 /* Skip the check for QP1 to support CM size of 128 */
2198 if (attrs->cap.max_send_wr > qattr->max_wqe) {
2199 QL_DPRINT12(ha, "cannot create a SQ with %d elements "
2200 " (max_send_wr=0x%x)\n",
2201 attrs->cap.max_send_wr, qattr->max_wqe);
2204 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {
2205 QL_DPRINT12(ha, "cannot create a RQ with %d elements"
2206 " (max_recv_wr=0x%x)\n",
2207 attrs->cap.max_recv_wr, qattr->max_wqe);
2210 if (attrs->cap.max_inline_data > qattr->max_inline) {
2212 "unsupported inline data size=0x%x "
2213 "requested (max_inline=0x%x)\n",
2214 attrs->cap.max_inline_data, qattr->max_inline);
2217 if (attrs->cap.max_send_sge > qattr->max_sge) {
2219 "unsupported send_sge=0x%x "
2220 "requested (max_send_sge=0x%x)\n",
2221 attrs->cap.max_send_sge, qattr->max_sge);
2224 if (attrs->cap.max_recv_sge > qattr->max_sge) {
2226 "unsupported recv_sge=0x%x requested "
2227 " (max_recv_sge=0x%x)\n",
2228 attrs->cap.max_recv_sge, qattr->max_sge);
2231 /* unprivileged user space cannot create special QP */
2232 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
2234 "userspace can't create special QPs of type=0x%x\n",
2238 /* allow creating only one GSI type of QP */
2239 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
2241 "create qp: GSI special QPs already created.\n");
2245 /* verify consumer QPs are not trying to use GSI QP's CQ */
2246 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
2247 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);
2248 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);
2250 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||
2251 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {
2252 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n");
2256 QL_DPRINT12(ha, "exit\n");
2261 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
2262 struct qlnxr_srq *srq,
2263 struct ib_udata *udata)
2265 struct qlnxr_create_srq_uresp uresp;
2271 QL_DPRINT12(ha, "enter\n");
2273 memset(&uresp, 0, sizeof(uresp));
2275 uresp.srq_id = srq->srq_id;
2277 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2279 QL_DPRINT12(ha, "exit [%d]\n", rc);
2284 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev,
2285 struct qlnxr_create_qp_uresp *uresp,
2286 struct qlnxr_qp *qp)
2292 /* Return if QP is associated with SRQ instead of RQ */
2293 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);
2298 /* iWARP requires two doorbells per RQ. */
2299 if (QLNX_IS_IWARP(dev)) {
2301 uresp->rq_db_offset =
2302 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2303 uresp->rq_db2_offset =
2304 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2306 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "
2307 "uresp->rq_db2_offset = 0x%x\n",
2308 uresp->rq_db_offset, uresp->rq_db2_offset);
2310 uresp->rq_db_offset =
2311 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2313 uresp->rq_icid = qp->icid;
2315 QL_DPRINT12(ha, "exit\n");
2320 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev,
2321 struct qlnxr_create_qp_uresp *uresp,
2322 struct qlnxr_qp *qp)
2328 QL_DPRINT12(ha, "enter\n");
2330 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2332 /* iWARP uses the same cid for rq and sq*/
2333 if (QLNX_IS_IWARP(dev)) {
2334 uresp->sq_icid = qp->icid;
2335 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);
2337 uresp->sq_icid = qp->icid + 1;
2339 QL_DPRINT12(ha, "exit\n");
2344 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev,
2345 struct qlnxr_qp *qp,
2346 struct ib_udata *udata)
2349 struct qlnxr_create_qp_uresp uresp;
2354 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);
2356 memset(&uresp, 0, sizeof(uresp));
2357 qlnxr_copy_sq_uresp(dev, &uresp, qp);
2358 qlnxr_copy_rq_uresp(dev, &uresp, qp);
2360 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
2361 uresp.qp_id = qp->qp_id;
2363 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2365 QL_DPRINT12(ha, "exit [%d]\n", rc);
2371 qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
2372 struct qlnxr_qp *qp,
2373 struct qlnxr_pd *pd,
2374 struct ib_qp_init_attr *attrs)
2380 QL_DPRINT12(ha, "enter\n");
2382 spin_lock_init(&qp->q_lock);
2384 atomic_set(&qp->refcnt, 1);
2386 qp->sig = QLNXR_QP_MAGIC_NUMBER;
2387 qp->qp_type = attrs->qp_type;
2388 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2389 qp->sq.max_sges = attrs->cap.max_send_sge;
2390 qp->state = ECORE_ROCE_QP_STATE_RESET;
2391 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
2392 qp->sq_cq = get_qlnxr_cq(attrs->send_cq);
2393 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);
2397 /* QP is associated with RQ instead of SRQ */
2398 qp->rq.max_sges = attrs->cap.max_recv_sge;
2399 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
2400 qp->rq.max_sges, qp->rq_cq->icid);
2402 qp->srq = get_qlnxr_srq(attrs->srq);
2406 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d,"
2407 " state = %d, signaled = %d, use_srq=%d\n",
2408 pd->pd_id, qp->qp_type, qp->max_inline_data,
2409 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));
2410 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
2411 qp->sq.max_sges, qp->sq_cq->icid);
2416 qlnxr_check_srq_params(struct ib_pd *ibpd,
2417 struct qlnxr_dev *dev,
2418 struct ib_srq_init_attr *attrs)
2420 struct ecore_rdma_device *qattr;
2424 qattr = ecore_rdma_query_device(dev->rdma_ctx);
2426 QL_DPRINT12(ha, "enter\n");
2428 if (attrs->attr.max_wr > qattr->max_srq_wqe) {
2429 QL_DPRINT12(ha, "unsupported srq_wr=0x%x"
2430 " requested (max_srq_wr=0x%x)\n",
2431 attrs->attr.max_wr, qattr->max_srq_wr);
2435 if (attrs->attr.max_sge > qattr->max_sge) {
2437 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
2438 attrs->attr.max_sge, qattr->max_sge);
2442 if (attrs->attr.srq_limit > attrs->attr.max_wr) {
2444 "unsupported srq_limit=0x%x requested"
2445 " (max_srq_limit=0x%x)\n",
2446 attrs->attr.srq_limit, attrs->attr.srq_limit);
2450 QL_DPRINT12(ha, "exit\n");
2456 qlnxr_free_srq_user_params(struct qlnxr_srq *srq)
2458 struct qlnxr_dev *dev = srq->dev;
2463 QL_DPRINT12(ha, "enter\n");
2465 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2466 ib_umem_release(srq->usrq.umem);
2467 ib_umem_release(srq->prod_umem);
2469 QL_DPRINT12(ha, "exit\n");
2474 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq)
2476 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2477 struct qlnxr_dev *dev = srq->dev;
2482 QL_DPRINT12(ha, "enter\n");
2484 ecore_chain_free(dev->cdev, &hw_srq->pbl);
2486 qlnx_dma_free_coherent(&dev->cdev,
2487 hw_srq->virt_prod_pair_addr,
2488 hw_srq->phy_prod_pair_addr,
2489 sizeof(struct rdma_srq_producers));
2491 QL_DPRINT12(ha, "exit\n");
2497 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
2498 struct qlnxr_srq *srq,
2499 struct qlnxr_create_srq_ureq *ureq,
2500 int access, int dmasync)
2502 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2503 struct ib_umem_chunk *chunk;
2505 struct scatterlist *sg;
2507 struct qlnxr_dev *dev = srq->dev;
2512 QL_DPRINT12(ha, "enter\n");
2514 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
2515 ureq->srq_len, access, dmasync, 1);
2519 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
2520 sizeof(struct rdma_srq_producers),
2522 if (IS_ERR(srq->prod_umem)) {
2524 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2525 ib_umem_release(srq->usrq.umem);
2527 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n",
2528 PTR_ERR(srq->prod_umem));
2530 return PTR_ERR(srq->prod_umem);
2533 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2534 chunk = container_of((&srq->prod_umem->chunk_list)->next,
2535 typeof(*chunk), list);
2536 sg = &chunk->page_list[0];
2538 sg = srq->prod_umem->sg_head.sgl;
2540 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
2542 QL_DPRINT12(ha, "exit\n");
2548 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
2549 struct qlnxr_dev *dev,
2550 struct ib_srq_init_attr *init_attr)
2552 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq;
2553 dma_addr_t phy_prod_pair_addr;
2554 u32 num_elems, max_wr;
2561 QL_DPRINT12(ha, "enter\n");
2563 va = qlnx_dma_alloc_coherent(&dev->cdev,
2564 &phy_prod_pair_addr,
2565 sizeof(struct rdma_srq_producers));
2567 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n");
2571 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
2572 hw_srq->virt_prod_pair_addr = va;
2574 max_wr = init_attr->attr.max_wr;
2576 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE;
2578 rc = ecore_chain_alloc(dev->cdev,
2579 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2580 ECORE_CHAIN_MODE_PBL,
2581 ECORE_CHAIN_CNT_TYPE_U32,
2583 ECORE_RDMA_SRQ_WQE_ELEM_SIZE,
2584 &hw_srq->pbl, NULL);
2587 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);
2591 hw_srq->max_wr = max_wr;
2592 hw_srq->num_elems = num_elems;
2593 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;
2595 QL_DPRINT12(ha, "exit\n");
2599 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,
2600 sizeof(struct rdma_srq_producers));
2602 QL_DPRINT12(ha, "exit [%d]\n", rc);
2607 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev,
2608 struct qlnxr_pd *pd,
2609 struct qlnxr_qp *qp,
2610 struct ib_qp_init_attr *attrs,
2611 bool fmr_and_reserved_lkey,
2612 struct ecore_rdma_create_qp_in_params *params)
2618 QL_DPRINT12(ha, "enter\n");
2620 /* QP handle to be written in an async event */
2621 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
2622 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
2624 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
2625 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
2626 params->pd = pd->pd_id;
2627 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
2628 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
2629 params->stats_queue = 0;
2631 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
2634 /* QP is associated with SRQ instead of RQ */
2635 params->srq_id = qp->srq->srq_id;
2636 params->use_srq = true;
2637 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
2638 params->srq_id, params->use_srq);
2643 params->use_srq = false;
2645 QL_DPRINT12(ha, "exit\n");
2651 qlnxr_qp_user_print( struct qlnxr_dev *dev,
2652 struct qlnxr_qp *qp)
2654 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "
2655 "rq_addr=0x%llx, rq_len=%zd\n",
2656 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
2662 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id)
2670 QL_DPRINT12(ha, "enter\n");
2672 if (!QLNX_IS_IWARP(dev))
2676 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {
2677 QL_DPRINT11(ha, "idr_pre_get failed\n");
2681 mtx_lock(&dev->idr_lock);
2683 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);
2685 mtx_unlock(&dev->idr_lock);
2687 } while (rc == -EAGAIN);
2689 QL_DPRINT12(ha, "exit [%d]\n", rc);
2695 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id)
2701 QL_DPRINT12(ha, "enter\n");
2703 if (!QLNX_IS_IWARP(dev))
2706 mtx_lock(&dev->idr_lock);
2707 idr_remove(&dev->qpidr, id);
2708 mtx_unlock(&dev->idr_lock);
2710 QL_DPRINT12(ha, "exit \n");
2716 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev,
2717 struct qlnxr_qp *qp,
2718 struct ecore_rdma_create_qp_out_params *out_params)
2724 QL_DPRINT12(ha, "enter\n");
2726 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
2727 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
2729 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
2733 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);
2737 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
2738 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
2740 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
2743 QL_DPRINT12(ha, "exit\n");
2748 qlnxr_create_user_qp(struct qlnxr_dev *dev,
2749 struct qlnxr_qp *qp,
2751 struct ib_udata *udata,
2752 struct ib_qp_init_attr *attrs)
2754 struct ecore_rdma_destroy_qp_out_params d_out_params;
2755 struct ecore_rdma_create_qp_in_params in_params;
2756 struct ecore_rdma_create_qp_out_params out_params;
2757 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
2758 struct ib_ucontext *ib_ctx = NULL;
2759 struct qlnxr_ucontext *ctx = NULL;
2760 struct qlnxr_create_qp_ureq ureq;
2761 int alloc_and_init = QLNX_IS_ROCE(dev);
2767 QL_DPRINT12(ha, "enter\n");
2769 ib_ctx = ibpd->uobject->context;
2770 ctx = get_qlnxr_ucontext(ib_ctx);
2772 memset(&ureq, 0, sizeof(ureq));
2773 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
2776 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);
2780 /* SQ - read access only (0), dma sync not required (0) */
2781 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
2785 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2790 /* RQ - read access only (0), dma sync not required (0) */
2791 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
2796 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2801 memset(&in_params, 0, sizeof(in_params));
2802 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
2803 in_params.qp_handle_lo = ureq.qp_handle_lo;
2804 in_params.qp_handle_hi = ureq.qp_handle_hi;
2805 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
2806 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
2809 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
2810 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
2813 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);
2815 if (!qp->ecore_qp) {
2817 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
2821 if (QLNX_IS_IWARP(dev))
2822 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params);
2824 qp->qp_id = out_params.qp_id;
2825 qp->icid = out_params.icid;
2827 rc = qlnxr_copy_qp_uresp(dev, qp, udata);
2830 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n");
2834 qlnxr_qp_user_print(dev, qp);
2836 QL_DPRINT12(ha, "exit\n");
2839 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
2842 QL_DPRINT12(ha, "fatal fault\n");
2845 qlnxr_cleanup_user(dev, qp);
2847 QL_DPRINT12(ha, "exit[%d]\n", rc);
2852 qlnxr_set_roce_db_info(struct qlnxr_dev *dev,
2853 struct qlnxr_qp *qp)
2859 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2861 qp->sq.db = dev->db_addr +
2862 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2863 qp->sq.db_data.data.icid = qp->icid + 1;
2866 qp->rq.db = dev->db_addr +
2867 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2868 qp->rq.db_data.data.icid = qp->icid;
2871 QL_DPRINT12(ha, "exit\n");
2876 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev,
2877 struct qlnxr_qp *qp)
2884 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2886 qp->sq.db = dev->db_addr +
2887 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2888 qp->sq.db_data.data.icid = qp->icid;
2891 qp->rq.db = dev->db_addr +
2892 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2893 qp->rq.db_data.data.icid = qp->icid;
2895 qp->rq.iwarp_db2 = dev->db_addr +
2896 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2897 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2898 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2902 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"
2903 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"
2904 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"
2905 " qp->rq.iwarp_db2.data.prod_val =0x%x\n",
2906 qp->sq.db, qp->sq.db_data.data.icid,
2907 qp->rq.db, qp->rq.db_data.data.icid,
2908 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,
2909 qp->rq.iwarp_db2_data.data.value);
2911 QL_DPRINT12(ha, "exit\n");
2916 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev,
2917 struct qlnxr_qp *qp,
2918 struct ecore_rdma_create_qp_in_params *in_params,
2922 struct ecore_rdma_create_qp_out_params out_params;
2928 QL_DPRINT12(ha, "enter\n");
2930 rc = ecore_chain_alloc(
2932 ECORE_CHAIN_USE_TO_PRODUCE,
2933 ECORE_CHAIN_MODE_PBL,
2934 ECORE_CHAIN_CNT_TYPE_U32,
2936 QLNXR_SQE_ELEMENT_SIZE,
2941 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);
2945 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);
2946 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);
2950 rc = ecore_chain_alloc(
2952 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2953 ECORE_CHAIN_MODE_PBL,
2954 ECORE_CHAIN_CNT_TYPE_U32,
2956 QLNXR_RQE_ELEMENT_SIZE,
2962 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);
2966 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);
2967 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);
2970 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
2972 if (!qp->ecore_qp) {
2973 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");
2977 qp->qp_id = out_params.qp_id;
2978 qp->icid = out_params.icid;
2980 qlnxr_set_roce_db_info(dev, qp);
2982 QL_DPRINT12(ha, "exit\n");
2987 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev,
2988 struct qlnxr_qp *qp,
2989 struct ecore_rdma_create_qp_in_params *in_params,
2993 struct ecore_rdma_destroy_qp_out_params d_out_params;
2994 struct ecore_rdma_create_qp_out_params out_params;
2995 struct ecore_chain_ext_pbl ext_pbl;
3001 QL_DPRINT12(ha, "enter\n");
3003 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,
3004 QLNXR_SQE_ELEMENT_SIZE,
3005 ECORE_CHAIN_MODE_PBL);
3006 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,
3007 QLNXR_RQE_ELEMENT_SIZE,
3008 ECORE_CHAIN_MODE_PBL);
3010 QL_DPRINT12(ha, "n_sq_elems = 0x%x"
3011 " n_rq_elems = 0x%x in_params\n"
3012 "\t\t\tqp_handle_lo\t\t= 0x%08x\n"
3013 "\t\t\tqp_handle_hi\t\t= 0x%08x\n"
3014 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"
3015 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"
3016 "\t\t\tuse_srq\t\t\t= 0x%x\n"
3017 "\t\t\tsignal_all\t\t= 0x%x\n"
3018 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"
3019 "\t\t\tpd\t\t\t= 0x%x\n"
3020 "\t\t\tdpi\t\t\t= 0x%x\n"
3021 "\t\t\tsq_cq_id\t\t\t= 0x%x\n"
3022 "\t\t\tsq_num_pages\t\t= 0x%x\n"
3023 "\t\t\tsq_pbl_ptr\t\t= %p\n"
3024 "\t\t\tmax_sq_sges\t\t= 0x%x\n"
3025 "\t\t\trq_cq_id\t\t\t= 0x%x\n"
3026 "\t\t\trq_num_pages\t\t= 0x%x\n"
3027 "\t\t\trq_pbl_ptr\t\t= %p\n"
3028 "\t\t\tsrq_id\t\t\t= 0x%x\n"
3029 "\t\t\tstats_queue\t\t= 0x%x\n",
3030 n_sq_elems, n_rq_elems,
3031 in_params->qp_handle_lo,
3032 in_params->qp_handle_hi,
3033 in_params->qp_handle_async_lo,
3034 in_params->qp_handle_async_hi,
3036 in_params->signal_all,
3037 in_params->fmr_and_reserved_lkey,
3040 in_params->sq_cq_id,
3041 in_params->sq_num_pages,
3042 (void *)in_params->sq_pbl_ptr,
3043 in_params->max_sq_sges,
3044 in_params->rq_cq_id,
3045 in_params->rq_num_pages,
3046 (void *)in_params->rq_pbl_ptr,
3048 in_params->stats_queue );
3050 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params));
3051 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl));
3053 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
3055 if (!qp->ecore_qp) {
3056 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
3060 /* Now we allocate the chain */
3061 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
3062 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
3064 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3065 "ext_pbl.p_pbl_phys = %p\n",
3066 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3068 rc = ecore_chain_alloc(
3070 ECORE_CHAIN_USE_TO_PRODUCE,
3071 ECORE_CHAIN_MODE_PBL,
3072 ECORE_CHAIN_CNT_TYPE_U32,
3074 QLNXR_SQE_ELEMENT_SIZE,
3080 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);
3084 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
3085 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
3087 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3088 "ext_pbl.p_pbl_phys = %p\n",
3089 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3093 rc = ecore_chain_alloc(
3095 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
3096 ECORE_CHAIN_MODE_PBL,
3097 ECORE_CHAIN_CNT_TYPE_U32,
3099 QLNXR_RQE_ELEMENT_SIZE,
3104 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"
3105 " failed rc = %d\n", rc);
3110 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",
3111 out_params.qp_id, out_params.icid);
3113 qp->qp_id = out_params.qp_id;
3114 qp->icid = out_params.icid;
3116 qlnxr_set_iwarp_db_info(dev, qp);
3118 QL_DPRINT12(ha, "exit\n");
3122 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
3124 QL_DPRINT12(ha, "exit rc = %d\n", rc);
3129 qlnxr_create_kernel_qp(struct qlnxr_dev *dev,
3130 struct qlnxr_qp *qp,
3132 struct ib_qp_init_attr *attrs)
3134 struct ecore_rdma_create_qp_in_params in_params;
3135 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3140 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3145 QL_DPRINT12(ha, "enter\n");
3147 memset(&in_params, 0, sizeof(in_params));
3149 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in
3150 * the ring. The ring should allow at least a single WR, even if the
3151 * user requested none, due to allocation issues.
3152 * We should add an extra WR since the prod and cons indices of
3153 * wqe_wr_id are managed in such a way that the WQ is considered full
3154 * when (prod+1)%max_wr==cons. We currently don't do that because we
3155 * double the number of entries due an iSER issue that pushes far more
3156 * WRs than indicated. If we decline its ib_post_send() then we get
3157 * error prints in the dmesg we'd like to avoid.
3159 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
3162 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
3164 if (!qp->wqe_wr_id) {
3165 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n");
3169 /* QP handle to be written in CQE */
3170 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp);
3171 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp);
3173 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in
3174 * the ring. There ring should allow at least a single WR, even if the
3175 * user requested none, due to allocation issues.
3177 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);
3179 /* Allocate driver internal RQ array */
3181 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
3183 if (!qp->rqe_wr_id) {
3184 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n");
3185 kfree(qp->wqe_wr_id);
3190 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
3192 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
3193 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
3195 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
3196 in_params.fmr_and_reserved_lkey = true;
3197 in_params.pd = pd->pd_id;
3198 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
3199 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
3200 in_params.stats_queue = 0;
3202 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
3205 /* QP is associated with SRQ instead of RQ */
3206 in_params.srq_id = qp->srq->srq_id;
3207 in_params.use_srq = true;
3208 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
3209 in_params.srq_id, in_params.use_srq);
3211 in_params.srq_id = 0;
3212 in_params.use_srq = false;
3215 n_sq_entries = attrs->cap.max_send_wr;
3216 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);
3217 n_sq_entries = max_t(u32, n_sq_entries, 1);
3218 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
3220 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
3222 if (QLNX_IS_ROCE(dev)) {
3223 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,
3224 n_sq_elems, n_rq_elems);
3226 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,
3227 n_sq_elems, n_rq_elems);
3231 qlnxr_cleanup_kernel(dev, qp);
3233 QL_DPRINT12(ha, "exit [%d]\n", rc);
3238 qlnxr_create_qp(struct ib_pd *ibpd,
3239 struct ib_qp_init_attr *attrs,
3240 struct ib_udata *udata)
3242 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);
3243 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3244 struct qlnxr_qp *qp;
3250 QL_DPRINT12(ha, "enter\n");
3252 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);
3254 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);
3258 QL_DPRINT12(ha, "called from %s, event_handle=%p,"
3259 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
3260 (udata ? "user library" : "kernel"),
3261 attrs->event_handler, pd,
3262 get_qlnxr_cq(attrs->send_cq),
3263 get_qlnxr_cq(attrs->send_cq)->icid,
3264 get_qlnxr_cq(attrs->recv_cq),
3265 get_qlnxr_cq(attrs->recv_cq)->icid);
3267 qp = qlnx_zalloc(sizeof(struct qlnxr_qp));
3270 QL_DPRINT11(ha, "kzalloc(qp) failed\n");
3271 return ERR_PTR(-ENOMEM);
3274 qlnxr_set_common_qp_params(dev, qp, pd, attrs);
3276 if (attrs->qp_type == IB_QPT_GSI) {
3277 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n");
3278 return qlnxr_create_gsi_qp(dev, attrs, qp);
3282 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);
3285 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n");
3289 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);
3292 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n");
3297 qp->ibqp.qp_num = qp->qp_id;
3299 rc = qlnxr_idr_add(dev, qp, qp->qp_id);
3302 QL_DPRINT11(ha, "qlnxr_idr_add failed\n");
3306 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);
3312 QL_DPRINT12(ha, "failed exit\n");
3313 return ERR_PTR(-EFAULT);
3317 static enum ib_qp_state
3318 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state)
3320 enum ib_qp_state state = IB_QPS_ERR;
3323 case ECORE_ROCE_QP_STATE_RESET:
3324 state = IB_QPS_RESET;
3327 case ECORE_ROCE_QP_STATE_INIT:
3328 state = IB_QPS_INIT;
3331 case ECORE_ROCE_QP_STATE_RTR:
3335 case ECORE_ROCE_QP_STATE_RTS:
3339 case ECORE_ROCE_QP_STATE_SQD:
3343 case ECORE_ROCE_QP_STATE_ERR:
3347 case ECORE_ROCE_QP_STATE_SQE:
3354 static enum ecore_roce_qp_state
3355 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state)
3357 enum ecore_roce_qp_state ecore_qp_state;
3359 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3363 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET;
3367 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT;
3371 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR;
3375 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS;
3379 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD;
3383 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3387 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3391 return (ecore_qp_state);
3395 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph)
3397 ecore_chain_reset(&qph->pbl);
3398 qph->prod = qph->cons = 0;
3400 qph->db_data.data.value = cpu_to_le16(0);
3406 qlnxr_update_qp_state(struct qlnxr_dev *dev,
3407 struct qlnxr_qp *qp,
3408 enum ecore_roce_qp_state new_state)
3412 struct ecore_dev *cdev;
3418 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",
3419 qp, new_state, qp->state);
3421 if (new_state == qp->state) {
3425 switch (qp->state) {
3426 case ECORE_ROCE_QP_STATE_RESET:
3427 switch (new_state) {
3428 case ECORE_ROCE_QP_STATE_INIT:
3429 qp->prev_wqe_size = 0;
3430 qlnxr_reset_qp_hwq_info(&qp->sq);
3432 qlnxr_reset_qp_hwq_info(&qp->rq);
3439 case ECORE_ROCE_QP_STATE_INIT:
3441 switch (new_state) {
3442 case ECORE_ROCE_QP_STATE_RTR:
3443 /* Update doorbell (in case post_recv was done before move to RTR) */
3447 //writel(qp->rq.db_data.raw, qp->rq.db);
3448 //if (QLNX_IS_IWARP(dev))
3449 // writel(qp->rq.iwarp_db2_data.raw,
3450 // qp->rq.iwarp_db2);
3452 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
3453 (uint8_t *)cdev->doorbells);
3455 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
3456 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
3458 if (QLNX_IS_IWARP(dev)) {
3459 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
3460 (uint8_t *)cdev->doorbells);
3461 bus_write_4(ha->pci_dbells, reg_addr,\
3462 qp->rq.iwarp_db2_data.raw);
3463 bus_barrier(ha->pci_dbells, 0, 0,\
3464 BUS_SPACE_BARRIER_READ);
3470 case ECORE_ROCE_QP_STATE_ERR:
3471 /* TBD:flush qps... */
3474 /* invalid state change. */
3479 case ECORE_ROCE_QP_STATE_RTR:
3481 switch (new_state) {
3482 case ECORE_ROCE_QP_STATE_RTS:
3484 case ECORE_ROCE_QP_STATE_ERR:
3487 /* invalid state change. */
3492 case ECORE_ROCE_QP_STATE_RTS:
3494 switch (new_state) {
3495 case ECORE_ROCE_QP_STATE_SQD:
3497 case ECORE_ROCE_QP_STATE_ERR:
3500 /* invalid state change. */
3505 case ECORE_ROCE_QP_STATE_SQD:
3507 switch (new_state) {
3508 case ECORE_ROCE_QP_STATE_RTS:
3509 case ECORE_ROCE_QP_STATE_ERR:
3512 /* invalid state change. */
3517 case ECORE_ROCE_QP_STATE_ERR:
3519 switch (new_state) {
3520 case ECORE_ROCE_QP_STATE_RESET:
3521 if ((qp->rq.prod != qp->rq.cons) ||
3522 (qp->sq.prod != qp->sq.cons)) {
3524 "Error->Reset with rq/sq "
3525 "not empty rq.prod=0x%x rq.cons=0x%x"
3526 " sq.prod=0x%x sq.cons=0x%x\n",
3527 qp->rq.prod, qp->rq.cons,
3528 qp->sq.prod, qp->sq.cons);
3542 QL_DPRINT12(ha, "exit\n");
3547 qlnxr_modify_qp(struct ib_qp *ibqp,
3548 struct ib_qp_attr *attr,
3550 struct ib_udata *udata)
3553 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3554 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);
3555 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
3556 enum ib_qp_state old_qp_state, new_qp_state;
3557 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3563 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",
3564 qp, attr_mask, attr->qp_state, udata);
3566 old_qp_state = qlnxr_get_ibqp_state(qp->state);
3567 if (attr_mask & IB_QP_STATE)
3568 new_qp_state = attr->qp_state;
3570 new_qp_state = old_qp_state;
3572 if (QLNX_IS_ROCE(dev)) {
3573 #if __FreeBSD_version >= 1100000
3574 if (!ib_modify_qp_is_ok(old_qp_state,
3578 IB_LINK_LAYER_ETHERNET)) {
3580 "invalid attribute mask=0x%x"
3581 " specified for qpn=0x%x of type=0x%x \n"
3582 " old_qp_state=0x%x, new_qp_state=0x%x\n",
3583 attr_mask, qp->qp_id, ibqp->qp_type,
3584 old_qp_state, new_qp_state);
3589 if (!ib_modify_qp_is_ok(old_qp_state,
3594 "invalid attribute mask=0x%x"
3595 " specified for qpn=0x%x of type=0x%x \n"
3596 " old_qp_state=0x%x, new_qp_state=0x%x\n",
3597 attr_mask, qp->qp_id, ibqp->qp_type,
3598 old_qp_state, new_qp_state);
3603 #endif /* #if __FreeBSD_version >= 1100000 */
3605 /* translate the masks... */
3606 if (attr_mask & IB_QP_STATE) {
3607 SET_FIELD(qp_params.modify_flags,
3608 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
3609 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);
3612 // TBD consider changing ecore to be a flag as well...
3613 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
3614 qp_params.sqd_async = true;
3616 if (attr_mask & IB_QP_PKEY_INDEX) {
3617 SET_FIELD(qp_params.modify_flags,
3618 ECORE_ROCE_MODIFY_QP_VALID_PKEY,
3620 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {
3625 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT;
3628 if (attr_mask & IB_QP_QKEY) {
3629 qp->qkey = attr->qkey;
3632 /* tbd consider splitting in ecore.. */
3633 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3634 SET_FIELD(qp_params.modify_flags,
3635 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
3636 qp_params.incoming_rdma_read_en =
3637 attr->qp_access_flags & IB_ACCESS_REMOTE_READ;
3638 qp_params.incoming_rdma_write_en =
3639 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;
3640 qp_params.incoming_atomic_en =
3641 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;
3644 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
3645 if (attr_mask & IB_QP_PATH_MTU) {
3646 if (attr->path_mtu < IB_MTU_256 ||
3647 attr->path_mtu > IB_MTU_4096) {
3650 "Only MTU sizes of 256, 512, 1024,"
3651 " 2048 and 4096 are supported "
3652 " attr->path_mtu = [%d]\n",
3658 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
3660 iboe_get_mtu(dev->ha->ifp->if_mtu)));
3664 qp->mtu = ib_mtu_enum_to_int(
3665 iboe_get_mtu(dev->ha->ifp->if_mtu));
3666 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",
3670 SET_FIELD(qp_params.modify_flags,
3671 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR,
3674 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
3675 qp_params.flow_label = attr->ah_attr.grh.flow_label;
3676 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
3678 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
3680 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params);
3682 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);
3686 qp_params.use_local_mac = true;
3687 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
3689 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",
3690 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
3691 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
3692 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",
3693 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
3694 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
3696 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",
3697 qp_params.remote_mac_addr[0],
3698 qp_params.remote_mac_addr[1],
3699 qp_params.remote_mac_addr[2],
3700 qp_params.remote_mac_addr[3],
3701 qp_params.remote_mac_addr[4],
3702 qp_params.remote_mac_addr[5]);
3704 qp_params.mtu = qp->mtu;
3707 if (qp_params.mtu == 0) {
3708 /* stay with current MTU */
3710 qp_params.mtu = qp->mtu;
3712 qp_params.mtu = ib_mtu_enum_to_int(
3713 iboe_get_mtu(dev->ha->ifp->if_mtu));
3717 if (attr_mask & IB_QP_TIMEOUT) {
3718 SET_FIELD(qp_params.modify_flags, \
3719 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
3721 qp_params.ack_timeout = attr->timeout;
3722 if (attr->timeout) {
3725 /* 12.7.34 LOCAL ACK TIMEOUT
3726 * Value representing the transport (ACK) timeout for
3727 * use by the remote, expressed as (4.096 μS*2Local ACK
3730 /* We use 1UL since the temporal value may be overflow
3733 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
3734 qp_params.ack_timeout = temp; /* FW requires [msec] */
3737 qp_params.ack_timeout = 0; /* infinite */
3739 if (attr_mask & IB_QP_RETRY_CNT) {
3740 SET_FIELD(qp_params.modify_flags,\
3741 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
3742 qp_params.retry_cnt = attr->retry_cnt;
3745 if (attr_mask & IB_QP_RNR_RETRY) {
3746 SET_FIELD(qp_params.modify_flags,
3747 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT,
3749 qp_params.rnr_retry_cnt = attr->rnr_retry;
3752 if (attr_mask & IB_QP_RQ_PSN) {
3753 SET_FIELD(qp_params.modify_flags,
3754 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN,
3756 qp_params.rq_psn = attr->rq_psn;
3757 qp->rq_psn = attr->rq_psn;
3760 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3761 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {
3764 "unsupported max_rd_atomic=%d, supported=%d\n",
3765 attr->max_rd_atomic,
3766 qattr->max_qp_req_rd_atomic_resc);
3770 SET_FIELD(qp_params.modify_flags,
3771 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ,
3773 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
3776 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
3777 SET_FIELD(qp_params.modify_flags,
3778 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER,
3780 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
3783 if (attr_mask & IB_QP_SQ_PSN) {
3784 SET_FIELD(qp_params.modify_flags,
3785 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN,
3787 qp_params.sq_psn = attr->sq_psn;
3788 qp->sq_psn = attr->sq_psn;
3791 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3792 if (attr->max_dest_rd_atomic >
3793 qattr->max_qp_resp_rd_atomic_resc) {
3795 "unsupported max_dest_rd_atomic=%d, "
3797 attr->max_dest_rd_atomic,
3798 qattr->max_qp_resp_rd_atomic_resc);
3804 SET_FIELD(qp_params.modify_flags,
3805 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP,
3807 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
3810 if (attr_mask & IB_QP_DEST_QPN) {
3811 SET_FIELD(qp_params.modify_flags,
3812 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP,
3815 qp_params.dest_qp = attr->dest_qp_num;
3816 qp->dest_qp_num = attr->dest_qp_num;
3820 * Update the QP state before the actual ramrod to prevent a race with
3821 * fast path. Modifying the QP state to error will cause the device to
3822 * flush the CQEs and while polling the flushed CQEs will considered as
3823 * a potential issue if the QP isn't in error state.
3825 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&
3826 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR))
3827 qp->state = ECORE_ROCE_QP_STATE_ERR;
3829 if (qp->qp_type != IB_QPT_GSI)
3830 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
3832 if (attr_mask & IB_QP_STATE) {
3833 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
3834 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);
3835 qp->state = qp_params.new_state;
3839 QL_DPRINT12(ha, "exit\n");
3844 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params)
3846 int ib_qp_acc_flags = 0;
3848 if (params->incoming_rdma_write_en)
3849 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
3850 if (params->incoming_rdma_read_en)
3851 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
3852 if (params->incoming_atomic_en)
3853 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
3854 if (true) /* FIXME -> local write ?? */
3855 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
3857 return ib_qp_acc_flags;
3861 qlnxr_mtu_int_to_enum(u16 mtu)
3863 enum ib_mtu ib_mtu_size;
3867 ib_mtu_size = IB_MTU_256;
3871 ib_mtu_size = IB_MTU_512;
3875 ib_mtu_size = IB_MTU_1024;
3879 ib_mtu_size = IB_MTU_2048;
3883 ib_mtu_size = IB_MTU_4096;
3887 ib_mtu_size = IB_MTU_1024;
3890 return (ib_mtu_size);
3894 qlnxr_query_qp(struct ib_qp *ibqp,
3895 struct ib_qp_attr *qp_attr,
3897 struct ib_qp_init_attr *qp_init_attr)
3900 struct ecore_rdma_query_qp_out_params params;
3901 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3902 struct qlnxr_dev *dev = qp->dev;
3907 QL_DPRINT12(ha, "enter\n");
3909 memset(¶ms, 0, sizeof(params));
3911 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms);
3915 memset(qp_attr, 0, sizeof(*qp_attr));
3916 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3918 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);
3919 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);
3921 /* In some cases in iWARP qelr will ask for the state only */
3922 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) {
3923 QL_DPRINT11(ha, "only state requested\n");
3927 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);
3928 qp_attr->path_mig_state = IB_MIG_MIGRATED;
3929 qp_attr->rq_psn = params.rq_psn;
3930 qp_attr->sq_psn = params.sq_psn;
3931 qp_attr->dest_qp_num = params.dest_qp;
3933 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms);
3935 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "
3936 "path_mtu = %d qp_access_flags = 0x%x\n",
3937 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,
3938 qp_attr->qp_access_flags);
3940 qp_attr->cap.max_send_wr = qp->sq.max_wr;
3941 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
3942 qp_attr->cap.max_send_sge = qp->sq.max_sges;
3943 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
3944 qp_attr->cap.max_inline_data = qp->max_inline_data;
3945 qp_init_attr->cap = qp_attr->cap;
3947 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0],
3948 sizeof(qp_attr->ah_attr.grh.dgid.raw));
3950 qp_attr->ah_attr.grh.flow_label = params.flow_label;
3951 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
3952 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
3953 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
3955 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
3956 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */
3957 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */
3958 qp_attr->timeout = params.timeout;
3959 qp_attr->rnr_retry = params.rnr_retry;
3960 qp_attr->retry_cnt = params.retry_cnt;
3961 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
3962 qp_attr->pkey_index = params.pkey_index;
3963 qp_attr->port_num = 1; /* FIXME -> check this */
3964 qp_attr->ah_attr.src_path_bits = 0;
3965 qp_attr->ah_attr.static_rate = 0;
3966 qp_attr->alt_pkey_index = 0;
3967 qp_attr->alt_port_num = 0;
3968 qp_attr->alt_timeout = 0;
3969 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
3971 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;
3972 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
3973 qp_attr->max_rd_atomic = params.max_rd_atomic;
3974 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;
3976 QL_DPRINT12(ha, "max_inline_data=%d\n",
3977 qp_attr->cap.max_inline_data);
3980 QL_DPRINT12(ha, "exit\n");
3986 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3992 QL_DPRINT12(ha, "enter\n");
3995 ib_umem_release(qp->usq.umem);
3997 qp->usq.umem = NULL;
4000 ib_umem_release(qp->urq.umem);
4002 qp->urq.umem = NULL;
4004 QL_DPRINT12(ha, "exit\n");
4009 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
4015 QL_DPRINT12(ha, "enter\n");
4017 if (qlnxr_qp_has_sq(qp)) {
4018 QL_DPRINT12(ha, "freeing SQ\n");
4019 ha->qlnxr_debug = 1;
4020 // ecore_chain_free(dev->cdev, &qp->sq.pbl);
4021 ha->qlnxr_debug = 0;
4022 kfree(qp->wqe_wr_id);
4025 if (qlnxr_qp_has_rq(qp)) {
4026 QL_DPRINT12(ha, "freeing RQ\n");
4027 ha->qlnxr_debug = 1;
4028 // ecore_chain_free(dev->cdev, &qp->rq.pbl);
4029 ha->qlnxr_debug = 0;
4030 kfree(qp->rqe_wr_id);
4033 QL_DPRINT12(ha, "exit\n");
4038 qlnxr_free_qp_resources(struct qlnxr_dev *dev,
4039 struct qlnxr_qp *qp)
4043 struct ecore_rdma_destroy_qp_out_params d_out_params;
4047 QL_DPRINT12(ha, "enter\n");
4050 if (qp->qp_type != IB_QPT_GSI) {
4051 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4057 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4058 qlnxr_cleanup_user(dev, qp);
4060 qlnxr_cleanup_kernel(dev, qp);
4063 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4064 qlnxr_cleanup_user(dev, qp);
4066 qlnxr_cleanup_kernel(dev, qp);
4068 if (qp->qp_type != IB_QPT_GSI) {
4069 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4075 QL_DPRINT12(ha, "exit\n");
4080 qlnxr_destroy_qp(struct ib_qp *ibqp)
4082 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4083 struct qlnxr_dev *dev = qp->dev;
4085 struct ib_qp_attr attr;
4091 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);
4095 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |
4096 ECORE_ROCE_QP_STATE_ERR |
4097 ECORE_ROCE_QP_STATE_INIT))) {
4099 attr.qp_state = IB_QPS_ERR;
4100 attr_mask |= IB_QP_STATE;
4102 /* change the QP state to ERROR */
4103 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL);
4106 if (qp->qp_type == IB_QPT_GSI)
4107 qlnxr_destroy_gsi_qp(dev);
4111 qlnxr_free_qp_resources(dev, qp);
4113 if (atomic_dec_and_test(&qp->refcnt)) {
4114 /* TODO: only for iWARP? */
4115 qlnxr_idr_remove(dev, qp->qp_id);
4119 QL_DPRINT12(ha, "exit\n");
4124 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq)
4126 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
4130 sge_data_len(struct ib_sge *sg_list, int num_sge)
4133 for (i = 0; i < num_sge; i++)
4134 len += sg_list[i].length;
4139 swap_wqe_data64(u64 *p)
4143 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
4144 *p = cpu_to_be64(cpu_to_le64(*p));
4149 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev,
4150 struct qlnxr_qp *qp,
4152 struct ib_send_wr *wr,
4153 struct ib_send_wr **bad_wr,
4158 char *seg_prt, *wqe;
4159 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
4164 QL_DPRINT12(ha, "enter[%d]\n", data_size);
4166 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
4168 "Too much inline data in WR:[%d, %d]\n",
4169 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE);
4180 seg_prt = wqe = NULL;
4183 /* copy data inline */
4184 for (i = 0; i < wr->num_sge; i++) {
4185 u32 len = wr->sg_list[i].length;
4186 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
4191 /* new segment required */
4193 wqe = (char *)ecore_chain_produce(&qp->sq.pbl);
4195 seg_siz = sizeof(struct rdma_sq_common_wqe);
4199 /* calculate currently allowed length */
4200 cur = MIN(len, seg_siz);
4202 memcpy(seg_prt, src, cur);
4204 /* update segment variables */
4207 /* update sge variables */
4211 /* swap fully-completed segments */
4213 swap_wqe_data64((u64 *)wqe);
4217 /* swap last not completed segment */
4219 swap_wqe_data64((u64 *)wqe);
4221 QL_DPRINT12(ha, "exit\n");
4226 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp,
4227 u8 *wqe_size, struct ib_send_wr *wr)
4235 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);
4237 for (i = 0; i < wr->num_sge; i++) {
4238 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);
4240 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
4241 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
4242 sge->length = cpu_to_le32(wr->sg_list[i].length);
4243 data_size += wr->sg_list[i].length;
4247 *wqe_size += wr->num_sge;
4249 QL_DPRINT12(ha, "exit data_size = %d\n", data_size);
4254 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev,
4255 struct qlnxr_qp *qp,
4256 struct rdma_sq_rdma_wqe_1st *rwqe,
4257 struct rdma_sq_rdma_wqe_2nd *rwqe2,
4258 struct ib_send_wr *wr,
4259 struct ib_send_wr **bad_wr)
4266 QL_DPRINT12(ha, "enter\n");
4268 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
4269 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);
4271 if (wr->send_flags & IB_SEND_INLINE) {
4273 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
4274 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,
4275 wr, bad_wr, &rwqe->flags, flags);
4278 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);
4280 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4286 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev,
4287 struct qlnxr_qp *qp,
4288 struct rdma_sq_send_wqe *swqe,
4289 struct rdma_sq_send_wqe *swqe2,
4290 struct ib_send_wr *wr,
4291 struct ib_send_wr **bad_wr)
4298 QL_DPRINT12(ha, "enter\n");
4300 memset(swqe2, 0, sizeof(*swqe2));
4302 if (wr->send_flags & IB_SEND_INLINE) {
4304 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
4305 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,
4306 wr, bad_wr, &swqe->flags, flags);
4309 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);
4311 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4317 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info)
4323 int work = info->completed - info->completed_handled - 1;
4325 QL_DPRINT12(ha, "enter [%d]\n", work);
4327 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
4328 struct qlnxr_pbl *pbl;
4330 /* Free all the page list that are possible to be freed
4331 * (all the ones that were invalidated), under the assumption
4332 * that if an FMR was completed successfully that means that
4333 * if there was an invalidate operation before it also ended
4335 pbl = list_first_entry(&info->inuse_pbl_list,
4338 list_del(&pbl->list_entry);
4339 list_add_tail(&pbl->list_entry, &info->free_pbl_list);
4340 info->completed_handled++;
4343 QL_DPRINT12(ha, "exit\n");
4347 #if __FreeBSD_version >= 1102000
4349 static int qlnxr_prepare_reg(struct qlnxr_qp *qp,
4350 struct rdma_sq_fmr_wqe_1st *fwqe1,
4351 struct ib_reg_wr *wr)
4353 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);
4354 struct rdma_sq_fmr_wqe_2nd *fwqe2;
4356 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);
4357 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
4358 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
4359 fwqe1->l_key = wr->key;
4361 fwqe2->access_ctrl = 0;
4363 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
4364 !!(wr->access & IB_ACCESS_REMOTE_READ));
4365 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
4366 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
4367 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
4368 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
4369 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
4370 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
4371 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
4372 fwqe2->fmr_ctrl = 0;
4374 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4375 ilog2(mr->ibmr.page_size) - 12);
4377 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */
4378 fwqe2->length_lo = mr->ibmr.length;
4379 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
4380 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
4382 qp->wqe_wr_id[qp->sq.prod].mr = mr;
4390 build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr,
4391 struct mr_info *info)
4395 int num_pbes, total_num_pbes = 0;
4396 struct regpair *pbe;
4397 struct qlnxr_pbl *pbl_tbl = info->pbl_table;
4398 struct qlnxr_pbl_info *pbl_info = &info->pbl_info;
4403 QL_DPRINT12(ha, "enter\n");
4405 pbe = (struct regpair *)pbl_tbl->va;
4408 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
4409 buf_addr = wr->wr.fast_reg.page_list->page_list[i];
4410 pbe->lo = cpu_to_le32((u32)buf_addr);
4411 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr));
4417 if (total_num_pbes == pbl_info->num_pbes)
4420 /* if the given pbl is full storing the pbes,
4424 (pbl_info->pbl_size / sizeof(u64))) {
4426 pbe = (struct regpair *)pbl_tbl->va;
4430 QL_DPRINT12(ha, "exit\n");
4436 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info)
4443 QL_DPRINT12(ha, "enter\n");
4445 if (info->completed == 0) {
4446 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n");
4451 qlnx_handle_completed_mrs(dev, info);
4453 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list);
4455 if (list_empty(&info->free_pbl_list)) {
4456 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info,
4459 info->pbl_table = list_first_entry(&info->free_pbl_list,
4462 list_del(&info->pbl_table->list_entry);
4465 if (!info->pbl_table)
4468 QL_DPRINT12(ha, "exit\n");
4473 qlnxr_prepare_fmr(struct qlnxr_qp *qp,
4474 struct rdma_sq_fmr_wqe_1st *fwqe1,
4475 struct ib_send_wr *wr)
4477 struct qlnxr_dev *dev = qp->dev;
4479 struct qlnxr_fast_reg_page_list *frmr_list =
4480 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list);
4481 struct rdma_sq_fmr_wqe *fwqe2 =
4482 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl);
4488 QL_DPRINT12(ha, "enter\n");
4490 if (wr->wr.fast_reg.page_list_len == 0)
4493 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info);
4497 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start);
4498 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start);
4499 fwqe1->l_key = wr->wr.fast_reg.rkey;
4501 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ,
4502 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ));
4503 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE,
4504 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE));
4505 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC,
4506 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC));
4507 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1);
4508 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE,
4509 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE));
4511 fwqe2->fmr_ctrl = 0;
4513 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4514 ilog2(1 << wr->wr.fast_reg.page_shift) - 12);
4515 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0);
4517 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */
4518 fwqe2->length_lo = wr->wr.fast_reg.length;
4519 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa);
4520 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa);
4522 /* produce another wqe for fwqe3 */
4523 ecore_chain_produce(&qp->sq.pbl);
4525 fbo = wr->wr.fast_reg.iova_start -
4526 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
4528 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x"
4529 " length = %x pbl_addr %x:%x\n",
4530 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey,
4531 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo,
4532 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo);
4534 build_frmr_pbes(dev, wr, &frmr_list->info);
4536 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list;
4538 QL_DPRINT12(ha, "exit\n");
4542 #endif /* #if __FreeBSD_version >= 1102000 */
4544 static enum ib_wc_opcode
4545 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
4548 case IB_WR_RDMA_WRITE:
4549 case IB_WR_RDMA_WRITE_WITH_IMM:
4550 return IB_WC_RDMA_WRITE;
4551 case IB_WR_SEND_WITH_IMM:
4553 case IB_WR_SEND_WITH_INV:
4555 case IB_WR_RDMA_READ:
4556 return IB_WC_RDMA_READ;
4557 case IB_WR_ATOMIC_CMP_AND_SWP:
4558 return IB_WC_COMP_SWAP;
4559 case IB_WR_ATOMIC_FETCH_AND_ADD:
4560 return IB_WC_FETCH_ADD;
4562 #if __FreeBSD_version >= 1102000
4564 return IB_WC_REG_MR;
4566 case IB_WR_FAST_REG_MR:
4567 return IB_WC_FAST_REG_MR;
4568 #endif /* #if __FreeBSD_version >= 1102000 */
4570 case IB_WR_LOCAL_INV:
4571 return IB_WC_LOCAL_INV;
4577 qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr)
4579 int wq_is_full, err_wr, pbl_is_full;
4580 struct qlnxr_dev *dev = qp->dev;
4585 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr);
4587 /* prevent SQ overflow and/or processing of a bad WR */
4588 err_wr = wr->num_sge > qp->sq.max_sges;
4589 wq_is_full = qlnxr_wq_is_full(&qp->sq);
4590 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <
4591 QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
4592 if (wq_is_full || err_wr || pbl_is_full) {
4594 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {
4596 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;
4599 "error: WQ is full. Post send on QP failed"
4600 " (this error appears only once) "
4601 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4602 qp, wr, qp->err_bitmap);
4606 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {
4608 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;
4611 "error: WQ is bad. Post send on QP failed"
4612 " (this error appears only once) "
4613 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4614 qp, wr, qp->err_bitmap);
4618 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {
4620 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;
4623 "error: WQ PBL is full. Post send on QP failed"
4624 " (this error appears only once) "
4625 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4626 qp, wr, qp->err_bitmap);
4630 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr);
4635 qlnxr_post_send(struct ib_qp *ibqp,
4636 struct ib_send_wr *wr,
4637 struct ib_send_wr **bad_wr)
4639 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device);
4640 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4641 unsigned long flags;
4642 int status = 0, rc = 0;
4650 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4653 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4656 if (qp->qp_type == IB_QPT_GSI)
4657 return qlnxr_gsi_post_send(ibqp, wr, bad_wr);
4659 spin_lock_irqsave(&qp->q_lock, flags);
4661 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&
4662 (qp->state != ECORE_ROCE_QP_STATE_ERR) &&
4663 (qp->state != ECORE_ROCE_QP_STATE_SQD)) {
4664 spin_unlock_irqrestore(&qp->q_lock, flags);
4666 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",
4667 qp->icid, qp->state);
4672 QL_DPRINT11(ha, "Got an empty post send???\n");
4676 struct rdma_sq_common_wqe *wqe;
4677 struct rdma_sq_send_wqe *swqe;
4678 struct rdma_sq_send_wqe *swqe2;
4679 struct rdma_sq_rdma_wqe_1st *rwqe;
4680 struct rdma_sq_rdma_wqe_2nd *rwqe2;
4681 struct rdma_sq_local_inv_wqe *iwqe;
4682 struct rdma_sq_atomic_wqe *awqe1;
4683 struct rdma_sq_atomic_wqe *awqe2;
4684 struct rdma_sq_atomic_wqe *awqe3;
4685 struct rdma_sq_fmr_wqe_1st *fwqe1;
4687 if (!qlnxr_can_post_send(qp, wr)) {
4693 wqe = ecore_chain_produce(&qp->sq.pbl);
4695 qp->wqe_wr_id[qp->sq.prod].signaled =
4696 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
4700 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<
4701 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT);
4703 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \
4704 !!(wr->send_flags & IB_SEND_SOLICITED));
4706 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||
4709 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
4710 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \
4711 !!(wr->send_flags & IB_SEND_FENCE));
4713 wqe->prev_wqe_size = qp->prev_wqe_size;
4715 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);
4718 switch (wr->opcode) {
4720 case IB_WR_SEND_WITH_IMM:
4722 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
4723 swqe = (struct rdma_sq_send_wqe *)wqe;
4725 swqe2 = (struct rdma_sq_send_wqe *)
4726 ecore_chain_produce(&qp->sq.pbl);
4727 swqe->inv_key_or_imm_data =
4728 cpu_to_le32(wr->ex.imm_data);
4729 swqe->length = cpu_to_le32(
4730 qlnxr_prepare_sq_send_data(dev,
4731 qp, swqe, swqe2, wr,
4734 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4735 qp->prev_wqe_size = swqe->wqe_size;
4736 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4738 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",
4739 swqe->length, wr->ex.imm_data);
4745 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
4746 swqe = (struct rdma_sq_send_wqe *)wqe;
4749 swqe2 = (struct rdma_sq_send_wqe *)
4750 ecore_chain_produce(&qp->sq.pbl);
4751 swqe->length = cpu_to_le32(
4752 qlnxr_prepare_sq_send_data(dev,
4753 qp, swqe, swqe2, wr,
4755 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4756 qp->prev_wqe_size = swqe->wqe_size;
4757 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4759 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n",
4764 case IB_WR_SEND_WITH_INV:
4766 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
4767 swqe = (struct rdma_sq_send_wqe *)wqe;
4768 swqe2 = (struct rdma_sq_send_wqe *)
4769 ecore_chain_produce(&qp->sq.pbl);
4771 swqe->inv_key_or_imm_data =
4772 cpu_to_le32(wr->ex.invalidate_rkey);
4773 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,
4774 qp, swqe, swqe2, wr, bad_wr));
4775 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4776 qp->prev_wqe_size = swqe->wqe_size;
4777 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4779 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n",
4783 case IB_WR_RDMA_WRITE_WITH_IMM:
4785 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
4786 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4789 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
4790 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4791 ecore_chain_produce(&qp->sq.pbl);
4792 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4793 qp, rwqe, rwqe2, wr, bad_wr));
4794 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4795 qp->prev_wqe_size = rwqe->wqe_size;
4796 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4799 "RDMA WRITE w/ IMM length = %d imm data=%x\n",
4800 rwqe->length, rwqe->imm_data);
4804 case IB_WR_RDMA_WRITE:
4806 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
4807 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4810 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4811 ecore_chain_produce(&qp->sq.pbl);
4812 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4813 qp, rwqe, rwqe2, wr, bad_wr));
4814 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4815 qp->prev_wqe_size = rwqe->wqe_size;
4816 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4819 "RDMA WRITE w/o IMM length = %d\n",
4824 case IB_WR_RDMA_READ_WITH_INV:
4827 "RDMA READ WITH INVALIDATE not supported\n");
4834 case IB_WR_RDMA_READ:
4836 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
4837 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4840 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4841 ecore_chain_produce(&qp->sq.pbl);
4842 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4843 qp, rwqe, rwqe2, wr, bad_wr));
4845 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4846 qp->prev_wqe_size = rwqe->wqe_size;
4847 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4849 QL_DPRINT12(ha, "RDMA READ length = %d\n",
4854 case IB_WR_ATOMIC_CMP_AND_SWP:
4855 case IB_WR_ATOMIC_FETCH_AND_ADD:
4858 "ATOMIC operation = %s\n",
4859 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?
4860 "IB_WR_ATOMIC_CMP_AND_SWP" :
4861 "IB_WR_ATOMIC_FETCH_AND_ADD"));
4863 awqe1 = (struct rdma_sq_atomic_wqe *)wqe;
4864 awqe1->prev_wqe_size = 4;
4866 awqe2 = (struct rdma_sq_atomic_wqe *)
4867 ecore_chain_produce(&qp->sq.pbl);
4869 TYPEPTR_ADDR_SET(awqe2, remote_va, \
4870 atomic_wr(wr)->remote_addr);
4872 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
4874 awqe3 = (struct rdma_sq_atomic_wqe *)
4875 ecore_chain_produce(&qp->sq.pbl);
4877 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
4878 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
4879 TYPEPTR_ADDR_SET(awqe3, swap_data,
4880 atomic_wr(wr)->compare_add);
4882 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
4883 TYPEPTR_ADDR_SET(awqe3, swap_data,
4884 atomic_wr(wr)->swap);
4885 TYPEPTR_ADDR_SET(awqe3, cmp_data,
4886 atomic_wr(wr)->compare_add);
4889 qlnxr_prepare_sq_sges(dev, qp, NULL, wr);
4891 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;
4892 qp->prev_wqe_size = awqe1->prev_wqe_size;
4896 case IB_WR_LOCAL_INV:
4899 "INVALIDATE length (IB_WR_LOCAL_INV)\n");
4901 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
4902 iwqe->prev_wqe_size = 1;
4904 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
4905 iwqe->inv_l_key = wr->ex.invalidate_rkey;
4906 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;
4907 qp->prev_wqe_size = iwqe->prev_wqe_size;
4911 #if __FreeBSD_version >= 1102000
4915 QL_DPRINT12(ha, "IB_WR_REG_MR\n");
4917 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4918 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4919 fwqe1->wqe_size = 2;
4921 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));
4923 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);
4928 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
4929 qp->prev_wqe_size = fwqe1->wqe_size;
4933 case IB_WR_FAST_REG_MR:
4935 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n");
4937 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4938 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4939 fwqe1->prev_wqe_size = 3;
4941 rc = qlnxr_prepare_fmr(qp, fwqe1, wr);
4945 "FAST_MR (IB_WR_FAST_REG_MR) failed"
4951 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size;
4952 qp->prev_wqe_size = fwqe1->prev_wqe_size;
4955 #endif /* #if __FreeBSD_version >= 1102000 */
4959 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);
4968 * restore prod to its position before this WR was processed
4970 ecore_chain_set_prod(&qp->sq.pbl,
4971 le16_to_cpu(qp->sq.db_data.data.value),
4973 /* restore prev_wqe_size */
4974 qp->prev_wqe_size = wqe->prev_wqe_size;
4977 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr);
4978 break; /* out of the loop */
4981 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
4983 qlnxr_inc_sw_prod(&qp->sq);
4985 qp->sq.db_data.data.value++;
4991 * If there was a failure in the first WR then it will be triggered in
4992 * vane. However this is not harmful (as long as the producer value is
4993 * unchanged). For performance reasons we avoid checking for this
4994 * redundant doorbell.
4997 //writel(qp->sq.db_data.raw, qp->sq.db);
4999 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);
5000 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);
5001 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5005 spin_unlock_irqrestore(&qp->q_lock, flags);
5007 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
5014 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq)
5018 /* Calculate number of elements used based on producer
5019 * count and consumer count and subtract it from max
5020 * work request supported so that we get elements left.
5022 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
5024 return hw_srq->max_wr - used;
5029 qlnxr_post_recv(struct ib_qp *ibqp,
5030 struct ib_recv_wr *wr,
5031 struct ib_recv_wr **bad_wr)
5033 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
5034 struct qlnxr_dev *dev = qp->dev;
5035 unsigned long flags;
5042 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5045 QL_DPRINT12(ha, "enter\n");
5047 if (qp->qp_type == IB_QPT_GSI) {
5048 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");
5049 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr);
5053 QL_DPRINT11(ha, "qp->srq [%p]"
5054 " QP is associated with SRQ, cannot post RQ buffers\n",
5059 spin_lock_irqsave(&qp->q_lock, flags);
5061 if (qp->state == ECORE_ROCE_QP_STATE_RESET) {
5062 spin_unlock_irqrestore(&qp->q_lock, flags);
5065 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");
5073 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <
5074 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) ||
5075 (wr->num_sge > qp->rq.max_sges)) {
5080 for (i = 0; i < wr->num_sge; i++) {
5082 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5084 /* first one must include the number of SGE in the list */
5086 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);
5088 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);
5090 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \
5091 wr->sg_list[i].length, flags);
5093 /* Special case of no sges. FW requires between 1-4 sges...
5094 * in this case we need to post 1 sge with length zero. this is
5095 * because rdma write with immediate consumes an RQ. */
5098 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5100 /* first one must include the number of SGE in the list */
5101 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
5102 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
5104 //RQ_SGE_SET(rqe, 0, 0, flags);
5109 rqe->flags = cpu_to_le32(flags);
5114 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
5115 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
5117 qlnxr_inc_sw_prod(&qp->rq);
5121 qp->rq.db_data.data.value++;
5123 // writel(qp->rq.db_data.raw, qp->rq.db);
5125 // if (QLNX_IS_IWARP(dev)) {
5126 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
5127 // mmiowb(); /* for second doorbell */
5130 reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
5131 (uint8_t *)ha->cdev.doorbells);
5133 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
5134 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5136 if (QLNX_IS_IWARP(dev)) {
5137 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
5138 (uint8_t *)ha->cdev.doorbells);
5139 bus_write_4(ha->pci_dbells, reg_addr, \
5140 qp->rq.iwarp_db2_data.raw);
5141 bus_barrier(ha->pci_dbells, 0, 0, \
5142 BUS_SPACE_BARRIER_READ);
5148 spin_unlock_irqrestore(&qp->q_lock, flags);
5150 QL_DPRINT12(ha, "exit status = 0x%x\n", status);
5155 /* In fmr we need to increase the number of fmr completed counter for the fmr
5156 * algorithm determining whether we can free a pbl or not.
5157 * we need to perform this whether the work request was signaled or not. for
5158 * this purpose we call this function from the condition that checks if a wr
5159 * should be skipped, to make sure we don't miss it ( possibly this fmr
5160 * operation was not signalted)
5163 qlnxr_chk_if_fmr(struct qlnxr_qp *qp)
5165 #if __FreeBSD_version >= 1102000
5167 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
5168 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5170 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR)
5171 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5173 #endif /* #if __FreeBSD_version >= 1102000 */
5177 process_req(struct qlnxr_dev *dev,
5178 struct qlnxr_qp *qp,
5179 struct qlnxr_cq *cq,
5183 enum ib_wc_status status,
5187 qlnx_host_t *ha = dev->ha;
5189 QL_DPRINT12(ha, "enter\n");
5191 while (num_entries && qp->sq.wqe_cons != hw_cons) {
5192 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
5193 qlnxr_chk_if_fmr(qp);
5199 wc->status = status;
5202 wc->src_qp = qp->id;
5206 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
5207 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
5209 switch (wc->opcode) {
5211 case IB_WC_RDMA_WRITE:
5213 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
5216 "opcode = IB_WC_RDMA_WRITE bytes = %d\n",
5217 qp->wqe_wr_id[qp->sq.cons].bytes_len);
5220 case IB_WC_COMP_SWAP:
5221 case IB_WC_FETCH_ADD:
5225 #if __FreeBSD_version >= 1102000
5227 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5230 case IB_WC_FAST_REG_MR:
5231 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5233 #endif /* #if __FreeBSD_version >= 1102000 */
5235 case IB_WC_RDMA_READ:
5238 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);
5241 ;//DP_ERR("TBD ERROR");
5248 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
5249 ecore_chain_consume(&qp->sq.pbl);
5250 qlnxr_inc_sw_cons(&qp->sq);
5253 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5258 qlnxr_poll_cq_req(struct qlnxr_dev *dev,
5259 struct qlnxr_qp *qp,
5260 struct qlnxr_cq *cq,
5263 struct rdma_cqe_requester *req)
5266 qlnx_host_t *ha = dev->ha;
5268 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);
5270 switch (req->status) {
5272 case RDMA_CQE_REQ_STS_OK:
5274 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5278 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
5280 if (qp->state != ECORE_ROCE_QP_STATE_ERR)
5281 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5282 IB_WC_WR_FLUSH_ERR, 1);
5285 default: /* other errors case */
5287 /* process all WQE before the cosumer */
5288 qp->state = ECORE_ROCE_QP_STATE_ERR;
5289 cnt = process_req(dev, qp, cq, num_entries, wc,
5290 req->sq_cons - 1, IB_WC_SUCCESS, 0);
5292 /* if we have extra WC fill it with actual error info */
5294 if (cnt < num_entries) {
5295 enum ib_wc_status wc_status;
5297 switch (req->status) {
5298 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
5299 wc_status = IB_WC_BAD_RESP_ERR;
5301 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
5302 wc_status = IB_WC_LOC_LEN_ERR;
5304 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
5305 wc_status = IB_WC_LOC_QP_OP_ERR;
5307 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
5308 wc_status = IB_WC_LOC_PROT_ERR;
5310 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
5311 wc_status = IB_WC_MW_BIND_ERR;
5313 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
5314 wc_status = IB_WC_REM_INV_REQ_ERR;
5316 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
5317 wc_status = IB_WC_REM_ACCESS_ERR;
5319 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
5320 wc_status = IB_WC_REM_OP_ERR;
5322 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
5323 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
5325 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
5326 wc_status = IB_WC_RETRY_EXC_ERR;
5329 wc_status = IB_WC_GENERAL_ERR;
5332 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
5333 wc_status, 1 /* force use of WC */);
5337 QL_DPRINT12(ha, "exit cnt = %d\n", cnt);
5342 __process_resp_one(struct qlnxr_dev *dev,
5343 struct qlnxr_qp *qp,
5344 struct qlnxr_cq *cq,
5346 struct rdma_cqe_responder *resp,
5349 enum ib_wc_status wc_status = IB_WC_SUCCESS;
5350 #if __FreeBSD_version < 1102000
5353 qlnx_host_t *ha = dev->ha;
5355 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",
5358 wc->opcode = IB_WC_RECV;
5361 switch (resp->status) {
5363 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
5364 wc_status = IB_WC_LOC_ACCESS_ERR;
5367 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
5368 wc_status = IB_WC_LOC_LEN_ERR;
5371 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
5372 wc_status = IB_WC_LOC_QP_OP_ERR;
5375 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
5376 wc_status = IB_WC_LOC_PROT_ERR;
5379 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
5380 wc_status = IB_WC_MW_BIND_ERR;
5383 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
5384 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
5387 case RDMA_CQE_RESP_STS_OK:
5389 #if __FreeBSD_version >= 1102000
5390 if (resp->flags & QLNXR_RESP_IMM) {
5392 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5393 wc->wc_flags |= IB_WC_WITH_IMM;
5395 if (resp->flags & QLNXR_RESP_RDMA)
5396 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5398 if (resp->flags & QLNXR_RESP_INV) {
5400 "Invalid flags QLNXR_RESP_INV [0x%x]"
5401 "qp = %p qp->id = 0x%x cq = %p"
5402 " cq->icid = 0x%x\n",
5403 resp->flags, qp, qp->id, cq, cq->icid );
5405 } else if (resp->flags & QLNXR_RESP_INV) {
5407 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5408 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
5410 if (resp->flags & QLNXR_RESP_RDMA) {
5412 "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5413 "qp = %p qp->id = 0x%x cq = %p"
5414 " cq->icid = 0x%x\n",
5415 resp->flags, qp, qp->id, cq, cq->icid );
5417 } else if (resp->flags & QLNXR_RESP_RDMA) {
5418 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5419 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5420 resp->flags, qp, qp->id, cq, cq->icid );
5423 wc_status = IB_WC_SUCCESS;
5424 wc->byte_len = le32_to_cpu(resp->length);
5426 flags = resp->flags & QLNXR_RESP_RDMA_IMM;
5430 case QLNXR_RESP_RDMA_IMM:
5432 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5433 /* fall to set imm data */
5434 case QLNXR_RESP_IMM:
5436 le32_to_cpu(resp->imm_data_or_inv_r_Key);
5437 wc->wc_flags |= IB_WC_WITH_IMM;
5439 case QLNXR_RESP_RDMA:
5440 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5441 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5442 resp->flags, qp, qp->id, cq, cq->icid );
5445 /* valid configuration, but nothing todo here */
5448 #endif /* #if __FreeBSD_version >= 1102000 */
5452 wc_status = IB_WC_GENERAL_ERR;
5456 wc->status = wc_status;
5458 wc->src_qp = qp->id;
5462 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);
5468 process_resp_one_srq(struct qlnxr_dev *dev,
5469 struct qlnxr_qp *qp,
5470 struct qlnxr_cq *cq,
5472 struct rdma_cqe_responder *resp)
5474 struct qlnxr_srq *srq = qp->srq;
5476 qlnx_host_t *ha = dev->ha;
5478 QL_DPRINT12(ha, "enter\n");
5480 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);
5482 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5483 wc->status = IB_WC_WR_FLUSH_ERR;
5487 wc->src_qp = qp->id;
5491 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5494 /* PBL is maintained in case of WR granularity.
5495 * So increment WR consumer after consuming WR
5497 srq->hw_srq.wr_cons_cnt++;
5499 QL_DPRINT12(ha, "exit\n");
5504 process_resp_one(struct qlnxr_dev *dev,
5505 struct qlnxr_qp *qp,
5506 struct qlnxr_cq *cq,
5508 struct rdma_cqe_responder *resp)
5510 qlnx_host_t *ha = dev->ha;
5511 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5513 QL_DPRINT12(ha, "enter\n");
5515 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
5517 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5518 ecore_chain_consume(&qp->rq.pbl);
5519 qlnxr_inc_sw_cons(&qp->rq);
5521 QL_DPRINT12(ha, "exit\n");
5526 process_resp_flush(struct qlnxr_qp *qp,
5532 qlnx_host_t *ha = qp->dev->ha;
5534 QL_DPRINT12(ha, "enter\n");
5536 while (num_entries && qp->rq.wqe_cons != hw_cons) {
5538 wc->status = IB_WC_WR_FLUSH_ERR;
5541 wc->src_qp = qp->id;
5543 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5548 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5549 ecore_chain_consume(&qp->rq.pbl);
5550 qlnxr_inc_sw_cons(&qp->rq);
5553 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5558 try_consume_resp_cqe(struct qlnxr_cq *cq,
5559 struct qlnxr_qp *qp,
5560 struct rdma_cqe_responder *resp,
5563 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
5570 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev,
5571 struct qlnxr_qp *qp,
5572 struct qlnxr_cq *cq,
5575 struct rdma_cqe_responder *resp,
5579 qlnx_host_t *ha = dev->ha;
5581 QL_DPRINT12(ha, "enter\n");
5583 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
5587 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5592 qlnxr_poll_cq_resp(struct qlnxr_dev *dev,
5593 struct qlnxr_qp *qp,
5594 struct qlnxr_cq *cq,
5597 struct rdma_cqe_responder *resp,
5601 qlnx_host_t *ha = dev->ha;
5603 QL_DPRINT12(ha, "enter\n");
5605 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5606 cnt = process_resp_flush(qp, num_entries, wc,
5608 try_consume_resp_cqe(cq, qp, resp, update);
5610 cnt = process_resp_one(dev, qp, cq, wc, resp);
5615 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5620 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp,
5621 struct rdma_cqe_requester *req, int *update)
5623 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
5630 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags)
5633 qlnx_host_t *ha = dev->ha;
5635 QL_DPRINT12(ha, "enter\n");
5638 cq->db.data.agg_flags = flags;
5639 cq->db.data.value = cpu_to_le32(cons);
5641 reg_addr = (uint64_t)((uint8_t *)cq->db_addr -
5642 (uint8_t *)(ha->cdev.doorbells));
5644 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);
5645 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ);
5647 QL_DPRINT12(ha, "exit\n");
5651 // writeq(cq->db.raw, cq->db_addr);
5653 /* Note that since the FW allows 64 bit write only, in 32bit systems
5654 * the value of db_addr must be low enough. This is currently not
5657 // writel(cq->db.raw & 0xffffffff, cq->db_addr);
5664 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe)
5666 struct rdma_cqe_requester *resp_cqe = &cqe->req;
5667 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==
5672 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
5674 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5675 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
5677 union rdma_cqe *cqe = cq->latest_cqe;
5679 u32 old_cons, new_cons;
5680 unsigned long flags;
5681 qlnx_host_t *ha = dev->ha;
5683 QL_DPRINT12(ha, "enter\n");
5685 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5688 if (cq->destroyed) {
5689 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n",
5694 if (cq->cq_type == QLNXR_CQ_TYPE_GSI)
5695 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc);
5697 spin_lock_irqsave(&cq->cq_lock, flags);
5699 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5701 while (num_entries && is_valid_cqe(cq, cqe)) {
5703 struct qlnxr_qp *qp;
5704 struct rdma_cqe_requester *resp_cqe;
5705 enum rdma_cqe_type cqe_type;
5707 /* prevent speculative reads of any field of CQE */
5710 resp_cqe = &cqe->req;
5711 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,
5712 resp_cqe->qp_handle.lo);
5715 QL_DPRINT11(ha, "qp = NULL\n");
5721 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
5724 case RDMA_CQE_TYPE_REQUESTER:
5725 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries,
5727 try_consume_req_cqe(cq, qp, &cqe->req, &update);
5729 case RDMA_CQE_TYPE_RESPONDER_RQ:
5730 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries,
5731 wc, &cqe->resp, &update);
5733 case RDMA_CQE_TYPE_RESPONDER_SRQ:
5734 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries,
5735 wc, &cqe->resp, &update);
5737 case RDMA_CQE_TYPE_INVALID:
5739 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);
5746 cqe = cq->latest_cqe;
5748 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5750 cq->cq_cons += new_cons - old_cons;
5753 /* doorbell notifies abount latest VALID entry,
5754 * but chain already point to the next INVALID one
5756 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);
5757 QL_DPRINT12(ha, "cq = %p cons = 0x%x "
5758 "arm_flags = 0x%x db.icid = 0x%x\n", cq,
5759 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);
5762 spin_unlock_irqrestore(&cq->cq_lock, flags);
5764 QL_DPRINT12(ha, "exit\n");
5771 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
5773 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5774 unsigned long sflags;
5775 struct qlnxr_dev *dev;
5778 dev = get_qlnxr_dev((ibcq->device));
5781 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "
5782 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,
5783 flags, cq, cq->cq_cons, cq->cq_type);
5785 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5788 if (cq->destroyed) {
5789 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n",
5794 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {
5798 spin_lock_irqsave(&cq->cq_lock, sflags);
5802 if (flags & IB_CQ_SOLICITED) {
5803 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
5805 if (flags & IB_CQ_NEXT_COMP) {
5806 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
5809 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);
5811 spin_unlock_irqrestore(&cq->cq_lock, sflags);
5813 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);
5818 static struct qlnxr_mr *
5819 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
5821 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
5822 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
5823 struct qlnxr_mr *mr;
5829 QL_DPRINT12(ha, "enter ibpd = %p pd = %p "
5830 " pd_id = %d max_page_list_len = %d\n",
5831 ibpd, pd, pd->pd_id, max_page_list_len);
5833 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
5835 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
5840 mr->type = QLNXR_MR_FRMR;
5842 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,
5843 1 /* allow dual layer pbl */);
5845 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n");
5849 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
5851 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n");
5855 /* index only, 18 bit long, lkey = itid << 8 | key */
5856 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;
5858 mr->hw_mr.pd = pd->pd_id;
5859 mr->hw_mr.local_read = 1;
5860 mr->hw_mr.local_write = 0;
5861 mr->hw_mr.remote_read = 0;
5862 mr->hw_mr.remote_write = 0;
5863 mr->hw_mr.remote_atomic = 0;
5864 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
5865 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */
5866 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
5867 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
5869 mr->hw_mr.length = 0;
5870 mr->hw_mr.vaddr = 0;
5871 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
5872 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */
5873 mr->hw_mr.dma_mr = false;
5875 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
5877 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n");
5881 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
5882 mr->ibmr.rkey = mr->ibmr.lkey;
5884 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",
5890 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
5894 QL_DPRINT12(ha, "exit\n");
5899 #if __FreeBSD_version >= 1102000
5902 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
5904 struct qlnxr_dev *dev;
5905 struct qlnxr_mr *mr;
5908 dev = get_qlnxr_dev(ibpd->device);
5911 QL_DPRINT12(ha, "enter\n");
5913 if (mr_type != IB_MR_TYPE_MEM_REG)
5914 return ERR_PTR(-EINVAL);
5916 mr = __qlnxr_alloc_mr(ibpd, max_num_sg);
5919 return ERR_PTR(-EINVAL);
5921 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);
5927 qlnxr_set_page(struct ib_mr *ibmr, u64 addr)
5929 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5930 struct qlnxr_pbl *pbl_table;
5931 struct regpair *pbe;
5932 struct qlnxr_dev *dev;
5939 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
5940 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);
5944 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,
5947 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
5948 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
5949 pbe = (struct regpair *)pbl_table->va;
5950 pbe += mr->npages % pbes_in_page;
5951 pbe->lo = cpu_to_le32((u32)addr);
5952 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
5956 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,
5962 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
5963 int sg_nents, unsigned int *sg_offset)
5966 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5972 if (mr->dev == NULL)
5977 QL_DPRINT12(ha, "enter\n");
5980 qlnx_handle_completed_mrs(mr->dev, &mr->info);
5982 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page);
5984 QL_DPRINT12(ha, "exit ret = %d\n", ret);
5992 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
5994 struct qlnxr_dev *dev;
5995 struct qlnxr_mr *mr;
5997 struct ib_mr *ibmr = NULL;
5999 dev = get_qlnxr_dev((ibpd->device));
6002 QL_DPRINT12(ha, "enter\n");
6004 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len);
6007 ibmr = ERR_PTR(-EINVAL);
6012 QL_DPRINT12(ha, "exit %p\n", ibmr);
6017 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
6019 struct qlnxr_fast_reg_page_list *frmr_list;
6021 frmr_list = get_qlnxr_frmr_list(page_list);
6023 free_mr_info(frmr_list->dev, &frmr_list->info);
6025 kfree(frmr_list->ibfrpl.page_list);
6031 struct ib_fast_reg_page_list *
6032 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len)
6034 struct qlnxr_fast_reg_page_list *frmr_list = NULL;
6035 struct qlnxr_dev *dev;
6036 int size = page_list_len * sizeof(u64);
6040 dev = get_qlnxr_dev(ibdev);
6043 QL_DPRINT12(ha, "enter\n");
6045 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL);
6047 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n");
6051 frmr_list->dev = dev;
6052 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL);
6053 if (!frmr_list->ibfrpl.page_list) {
6054 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n");
6058 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len,
6059 1 /* allow dual layer pbl */);
6063 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl);
6065 return &frmr_list->ibfrpl;
6068 kfree(frmr_list->ibfrpl.page_list);
6072 QL_DPRINT12(ha, "exit with error\n");
6078 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6079 int buf_cnt, uint64_t *total_size)
6085 if (!buf_cnt || buf_list == NULL) {
6087 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt);
6091 size = buf_list->size;
6095 "failed buf_list = %p buf_cnt = %d"
6096 " buf_list->size = 0\n", buf_list, buf_cnt);
6102 *total_size += buf_list->size;
6104 if (buf_list->size != size) {
6106 "failed buf_list = %p buf_cnt = %d"
6107 " all buffers should have same size\n",
6119 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6123 size_t num_pages = 0;
6126 for (i = 0; i < buf_cnt; i++) {
6129 while (size < buf_list->size) {
6139 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev,
6140 struct ib_phys_buf *buf_list, int buf_cnt,
6141 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
6143 struct regpair *pbe;
6144 struct qlnxr_pbl *pbl_tbl;
6145 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
6152 QL_DPRINT12(ha, "enter\n");
6155 QL_DPRINT11(ha, "PBL_INFO not initialized\n");
6159 if (!pbl_info->num_pbes) {
6160 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
6164 /* If we have a two layered pbl, the first pbl points to the rest
6165 * of the pbls and the first entry lays on the second pbl in the table
6167 if (pbl_info->two_layered)
6172 pbe = (struct regpair *)pbl_tbl->va;
6174 QL_DPRINT12(ha, "pbe is NULL\n");
6180 for (i = 0; i < buf_cnt; i++) {
6182 pages = buf_list->size >> PAGE_SHIFT;
6184 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
6185 /* store the page address in pbe */
6187 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt);
6189 pbe->lo = cpu_to_le32((u32)pbe_addr);
6190 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32)));
6192 QL_DPRINT12(ha, "Populate pbl table:"
6193 " pbe->addr=0x%x:0x%x "
6194 " pbe_cnt = %d total_num_pbes=%d"
6195 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
6196 total_num_pbes, pbe);
6202 if (total_num_pbes == pbl_info->num_pbes)
6205 /* if the given pbl is full storing the pbes,
6206 * move to next pbl. */
6208 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
6210 pbe = (struct regpair *)pbl_tbl->va;
6216 QL_DPRINT12(ha, "exit\n");
6221 qlnxr_reg_kernel_mr(struct ib_pd *ibpd,
6222 struct ib_phys_buf *buf_list,
6223 int buf_cnt, int acc, u64 *iova_start)
6226 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
6227 struct qlnxr_mr *mr;
6228 struct qlnxr_pd *pd;
6230 size_t num_pages = 0;
6235 QL_DPRINT12(ha, "enter\n");
6237 pd = get_qlnxr_pd(ibpd);
6239 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d,"
6240 " iova_start = %p, acc = %d\n",
6241 pd->pd_id, buf_list, buf_cnt, iova_start, acc);
6243 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
6244 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&"
6245 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n");
6246 // return ERR_PTR(-EINVAL);
6249 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
6251 QL_DPRINT11(ha, "kzalloc(mr) failed\n");
6255 mr->type = QLNXR_MR_KERNEL;
6256 mr->iova_start = iova_start;
6258 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length);
6262 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt);
6266 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1);
6269 "qlnxr_init_mr_info failed [%d]\n", rc);
6273 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table,
6274 &mr->info.pbl_info);
6276 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
6279 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
6283 /* index only, 18 bit long, lkey = itid << 8 | key */
6284 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
6286 mr->hw_mr.pd = pd->pd_id;
6287 mr->hw_mr.local_read = 1;
6288 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
6289 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
6290 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
6291 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
6292 mr->hw_mr.mw_bind = false; /* TBD MW BIND */
6293 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
6294 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
6295 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
6296 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */
6300 mr->hw_mr.length = length;
6301 mr->hw_mr.vaddr = (uint64_t)iova_start;
6302 mr->hw_mr.zbva = false; /* TBD figure when this should be true */
6303 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
6304 mr->hw_mr.dma_mr = false;
6306 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
6308 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
6312 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6313 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
6314 mr->hw_mr.remote_atomic)
6315 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6317 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey);
6322 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
6324 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
6328 QL_DPRINT12(ha, "exit [%d]\n", rc);
6329 return (ERR_PTR(rc));
6332 #endif /* #if __FreeBSD_version >= 1102000 */
6335 #if __FreeBSD_version >= 1102000
6336 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
6337 struct ib_udata *udata)
6339 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
6340 #endif /* #if __FreeBSD_version >= 1102000 */
6342 struct qlnxr_dev *dev;
6344 struct qlnxr_ah *ah;
6346 dev = get_qlnxr_dev((ibpd->device));
6349 QL_DPRINT12(ha, "in create_ah\n");
6351 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
6353 QL_DPRINT12(ha, "no address handle can be allocated\n");
6354 return ERR_PTR(-ENOMEM);
6363 qlnxr_destroy_ah(struct ib_ah *ibah)
6365 struct qlnxr_dev *dev;
6367 struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
6369 dev = get_qlnxr_dev((ibah->device));
6372 QL_DPRINT12(ha, "in destroy_ah\n");
6379 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6381 struct qlnxr_dev *dev;
6384 dev = get_qlnxr_dev((ibah->device));
6386 QL_DPRINT12(ha, "Query AH not supported\n");
6391 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6393 struct qlnxr_dev *dev;
6396 dev = get_qlnxr_dev((ibah->device));
6398 QL_DPRINT12(ha, "Modify AH not supported\n");
6402 #if __FreeBSD_version >= 1102000
6404 qlnxr_process_mad(struct ib_device *ibdev,
6405 int process_mad_flags,
6407 const struct ib_wc *in_wc,
6408 const struct ib_grh *in_grh,
6409 const struct ib_mad_hdr *mad_hdr,
6411 struct ib_mad_hdr *out_mad,
6412 size_t *out_mad_size,
6413 u16 *out_mad_pkey_index)
6418 qlnxr_process_mad(struct ib_device *ibdev,
6419 int process_mad_flags,
6421 struct ib_wc *in_wc,
6422 struct ib_grh *in_grh,
6423 struct ib_mad *in_mad,
6424 struct ib_mad *out_mad)
6426 #endif /* #if __FreeBSD_version >= 1102000 */
6428 struct qlnxr_dev *dev;
6431 dev = get_qlnxr_dev(ibdev);
6433 QL_DPRINT12(ha, "process mad not supported\n");
6436 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",
6437 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,
6438 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,
6439 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,
6440 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);
6442 // return IB_MAD_RESULT_SUCCESS;
6446 #if __FreeBSD_version >= 1102000
6448 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
6449 struct ib_port_immutable *immutable)
6451 struct qlnxr_dev *dev;
6453 struct ib_port_attr attr;
6456 dev = get_qlnxr_dev(ibdev);
6459 QL_DPRINT12(ha, "enter\n");
6461 err = qlnxr_query_port(ibdev, port_num, &attr);
6465 if (QLNX_IS_IWARP(dev)) {
6466 immutable->pkey_tbl_len = 1;
6467 immutable->gid_tbl_len = 1;
6468 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
6469 immutable->max_mad_size = 0;
6471 immutable->pkey_tbl_len = attr.pkey_tbl_len;
6472 immutable->gid_tbl_len = attr.gid_tbl_len;
6473 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
6474 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
6477 QL_DPRINT12(ha, "exit\n");
6480 #endif /* #if __FreeBSD_version > 1102000 */
6483 /***** iWARP related functions *************/
6487 qlnxr_iw_mpa_request(void *context,
6488 struct ecore_iwarp_cm_event_params *params)
6490 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context;
6491 struct qlnxr_dev *dev = listener->dev;
6492 struct qlnxr_iw_ep *ep;
6493 struct iw_cm_event event;
6494 struct sockaddr_in *laddr;
6495 struct sockaddr_in *raddr;
6500 QL_DPRINT12(ha, "enter\n");
6502 if (params->cm_info->ip_version != ECORE_TCP_IPV4) {
6503 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",
6504 params->cm_info->ip_version);
6508 ep = kzalloc(sizeof(*ep), GFP_ATOMIC);
6511 QL_DPRINT11(ha, "kzalloc{ep) failed\n");
6516 ep->ecore_context = params->ep_context;
6518 memset(&event, 0, sizeof(event));
6520 event.event = IW_CM_EVENT_CONNECT_REQUEST;
6521 event.status = params->status;
6523 laddr = (struct sockaddr_in *)&event.local_addr;
6524 raddr = (struct sockaddr_in *)&event.remote_addr;
6526 laddr->sin_family = AF_INET;
6527 raddr->sin_family = AF_INET;
6529 laddr->sin_port = htons(params->cm_info->local_port);
6530 raddr->sin_port = htons(params->cm_info->remote_port);
6532 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);
6533 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);
6535 event.provider_data = (void *)ep;
6536 event.private_data = (void *)params->cm_info->private_data;
6537 event.private_data_len = (u8)params->cm_info->private_data_len;
6539 #if __FreeBSD_version >= 1100000
6540 event.ord = params->cm_info->ord;
6541 event.ird = params->cm_info->ird;
6542 #endif /* #if __FreeBSD_version >= 1100000 */
6544 listener->cm_id->event_handler(listener->cm_id, &event);
6546 QL_DPRINT12(ha, "exit\n");
6552 qlnxr_iw_issue_event(void *context,
6553 struct ecore_iwarp_cm_event_params *params,
6554 enum iw_cm_event_type event_type,
6557 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6558 struct qlnxr_dev *dev = ep->dev;
6559 struct iw_cm_event event;
6564 QL_DPRINT12(ha, "enter\n");
6566 memset(&event, 0, sizeof(event));
6567 event.status = params->status;
6568 event.event = event_type;
6570 if (params->cm_info != NULL) {
6571 #if __FreeBSD_version >= 1100000
6572 event.ird = params->cm_info->ird;
6573 event.ord = params->cm_info->ord;
6574 QL_DPRINT12(ha, "ord=[%d] \n", event.ord);
6575 QL_DPRINT12(ha, "ird=[%d] \n", event.ird);
6576 #endif /* #if __FreeBSD_version >= 1100000 */
6578 event.private_data_len = params->cm_info->private_data_len;
6579 event.private_data = (void *)params->cm_info->private_data;
6580 QL_DPRINT12(ha, "private_data_len=[%d] \n",
6581 event.private_data_len);
6584 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str);
6585 QL_DPRINT12(ha, "status=[%d] \n", event.status);
6589 ep->cm_id->event_handler(ep->cm_id, &event);
6591 QL_DPRINT11(ha, "ep->cm_id == NULL \n");
6593 QL_DPRINT11(ha, "ep == NULL \n");
6596 QL_DPRINT12(ha, "exit\n");
6602 qlnxr_iw_close_event(void *context,
6603 struct ecore_iwarp_cm_event_params *params)
6605 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6606 struct qlnxr_dev *dev = ep->dev;
6611 QL_DPRINT12(ha, "enter\n");
6614 qlnxr_iw_issue_event(context,
6617 "IW_CM_EVENT_EVENT_CLOSE");
6618 ep->cm_id->rem_ref(ep->cm_id);
6622 QL_DPRINT12(ha, "exit\n");
6627 #if __FreeBSD_version >= 1102000
6630 qlnxr_iw_passive_complete(void *context,
6631 struct ecore_iwarp_cm_event_params *params)
6633 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6634 struct qlnxr_dev *dev = ep->dev;
6639 /* We will only reach the following state if MPA_REJECT was called on
6640 * passive. In this case there will be no associated QP.
6642 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {
6643 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n");
6648 /* We always issue an established event, however, ofed does not look
6649 * at event code for established. So if there was a failure, we follow
6652 qlnxr_iw_issue_event(context,
6654 IW_CM_EVENT_ESTABLISHED,
6655 "IW_CM_EVENT_ESTABLISHED");
6657 if (params->status < 0) {
6658 qlnxr_iw_close_event(context, params);
6664 struct qlnxr_discon_work {
6665 struct work_struct work;
6666 struct qlnxr_iw_ep *ep;
6667 enum ecore_iwarp_event_type event;
6672 qlnxr_iw_disconnect_worker(struct work_struct *work)
6674 struct qlnxr_discon_work *dwork =
6675 container_of(work, struct qlnxr_discon_work, work);
6676 struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
6677 struct qlnxr_iw_ep *ep = dwork->ep;
6678 struct qlnxr_dev *dev = ep->dev;
6679 struct qlnxr_qp *qp = ep->qp;
6680 struct iw_cm_event event;
6682 if (qp->destroyed) {
6684 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6688 memset(&event, 0, sizeof(event));
6689 event.status = dwork->status;
6690 event.event = IW_CM_EVENT_DISCONNECT;
6692 /* Success means graceful disconnect was requested. modifying
6693 * to SQD is translated to graceful disconnect. O/w reset is sent
6696 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR;
6698 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD;
6703 ep->cm_id->event_handler(ep->cm_id, &event);
6705 SET_FIELD(qp_params.modify_flags,
6706 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
6708 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
6710 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6716 qlnxr_iw_disconnect_event(void *context,
6717 struct ecore_iwarp_cm_event_params *params)
6719 struct qlnxr_discon_work *work;
6720 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6721 struct qlnxr_dev *dev = ep->dev;
6722 struct qlnxr_qp *qp = ep->qp;
6724 work = kzalloc(sizeof(*work), GFP_ATOMIC);
6728 qlnxr_iw_qp_add_ref(&qp->ibqp);
6730 work->event = params->event;
6731 work->status = params->status;
6733 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);
6734 queue_work(dev->iwarp_wq, &work->work);
6739 #endif /* #if __FreeBSD_version >= 1102000 */
6742 qlnxr_iw_mpa_reply(void *context,
6743 struct ecore_iwarp_cm_event_params *params)
6745 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6746 struct qlnxr_dev *dev = ep->dev;
6747 struct ecore_iwarp_send_rtr_in rtr_in;
6753 QL_DPRINT12(ha, "enter\n");
6755 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6758 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in));
6759 rtr_in.ep_context = params->ep_context;
6761 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);
6763 QL_DPRINT12(ha, "exit rc = %d\n", rc);
6769 qlnxr_iw_qp_event(void *context,
6770 struct ecore_iwarp_cm_event_params *params,
6771 enum ib_event_type ib_event,
6774 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6775 struct qlnxr_dev *dev = ep->dev;
6776 struct ib_qp *ibqp = &(ep->qp->ibqp);
6777 struct ib_event event;
6783 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",
6784 context, params->event, str, ibqp->event_handler);
6786 if (ibqp->event_handler) {
6787 event.event = ib_event;
6788 event.device = ibqp->device;
6789 event.element.qp = ibqp;
6790 ibqp->event_handler(&event, ibqp->qp_context);
6797 qlnxr_iw_event_handler(void *context,
6798 struct ecore_iwarp_cm_event_params *params)
6800 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6801 struct qlnxr_dev *dev = ep->dev;
6806 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6807 "enter\n", context, params->event);
6809 switch (params->event) {
6811 /* Passive side request received */
6812 case ECORE_IWARP_EVENT_MPA_REQUEST:
6813 qlnxr_iw_mpa_request(context, params);
6816 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY:
6817 qlnxr_iw_mpa_reply(context, params);
6820 /* Passive side established ( ack on mpa response ) */
6821 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE:
6823 #if __FreeBSD_version >= 1102000
6825 ep->during_connect = 0;
6826 qlnxr_iw_passive_complete(context, params);
6829 qlnxr_iw_issue_event(context,
6831 IW_CM_EVENT_ESTABLISHED,
6832 "IW_CM_EVENT_ESTABLISHED");
6833 #endif /* #if __FreeBSD_version >= 1102000 */
6836 /* Active side reply received */
6837 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE:
6838 ep->during_connect = 0;
6839 qlnxr_iw_issue_event(context,
6841 IW_CM_EVENT_CONNECT_REPLY,
6842 "IW_CM_EVENT_CONNECT_REPLY");
6843 if (params->status < 0) {
6844 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6846 ep->cm_id->rem_ref(ep->cm_id);
6851 case ECORE_IWARP_EVENT_DISCONNECT:
6853 #if __FreeBSD_version >= 1102000
6854 qlnxr_iw_disconnect_event(context, params);
6856 qlnxr_iw_issue_event(context,
6858 IW_CM_EVENT_DISCONNECT,
6859 "IW_CM_EVENT_DISCONNECT");
6860 qlnxr_iw_close_event(context, params);
6861 #endif /* #if __FreeBSD_version >= 1102000 */
6864 case ECORE_IWARP_EVENT_CLOSE:
6865 ep->during_connect = 0;
6866 qlnxr_iw_close_event(context, params);
6869 case ECORE_IWARP_EVENT_RQ_EMPTY:
6870 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6871 "IWARP_EVENT_RQ_EMPTY");
6874 case ECORE_IWARP_EVENT_IRQ_FULL:
6875 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6876 "IWARP_EVENT_IRQ_FULL");
6879 case ECORE_IWARP_EVENT_LLP_TIMEOUT:
6880 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6881 "IWARP_EVENT_LLP_TIMEOUT");
6884 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR:
6885 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6886 "IWARP_EVENT_REMOTE_PROTECTION_ERROR");
6889 case ECORE_IWARP_EVENT_CQ_OVERFLOW:
6890 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6891 "QED_IWARP_EVENT_CQ_OVERFLOW");
6894 case ECORE_IWARP_EVENT_QP_CATASTROPHIC:
6895 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6896 "QED_IWARP_EVENT_QP_CATASTROPHIC");
6899 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR:
6900 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6901 "IWARP_EVENT_LOCAL_ACCESS_ERROR");
6904 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR:
6905 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6906 "IWARP_EVENT_REMOTE_OPERATION_ERROR");
6909 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED:
6910 QL_DPRINT12(ha, "Got terminate message"
6911 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n");
6916 "Unknown event [0x%x] received \n", params->event);
6920 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6921 "exit\n", context, params->event);
6926 qlnxr_addr4_resolve(struct qlnxr_dev *dev,
6927 struct sockaddr_in *src_in,
6928 struct sockaddr_in *dst_in,
6933 #if __FreeBSD_version >= 1100000
6934 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,
6935 dst_mac, NULL, NULL);
6937 struct llentry *lle;
6939 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in,
6943 QL_DPRINT12(dev->ha, "rc = %d "
6944 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "
6945 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,
6946 dst_in->sin_len, dst_in->sin_family,
6947 NIPQUAD((dst_in->sin_addr.s_addr)),
6948 dst_mac[0], dst_mac[1], dst_mac[2],
6949 dst_mac[3], dst_mac[4], dst_mac[5]);
6955 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
6957 struct qlnxr_dev *dev;
6958 struct ecore_iwarp_connect_out out_params;
6959 struct ecore_iwarp_connect_in in_params;
6960 struct qlnxr_iw_ep *ep;
6961 struct qlnxr_qp *qp;
6962 struct sockaddr_in *laddr;
6963 struct sockaddr_in *raddr;
6967 dev = get_qlnxr_dev((cm_id->device));
6970 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] "
6971 "enter \n", cm_id, conn_param);
6973 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6976 qp = idr_find(&dev->qpidr, conn_param->qpn);
6978 laddr = (struct sockaddr_in *)&cm_id->local_addr;
6979 raddr = (struct sockaddr_in *)&cm_id->remote_addr;
6982 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n",
6983 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,
6984 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);
6986 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
6988 QL_DPRINT11(ha, "struct qlnxr_iw_ep "
6989 "alloc memory failed\n");
6995 cm_id->add_ref(cm_id);
6998 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in));
6999 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out));
7001 in_params.event_cb = qlnxr_iw_event_handler;
7002 in_params.cb_context = ep;
7004 in_params.cm_info.ip_version = ECORE_TCP_IPV4;
7006 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);
7007 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);
7008 in_params.cm_info.remote_port = ntohs(raddr->sin_port);
7009 in_params.cm_info.local_port = ntohs(laddr->sin_port);
7010 in_params.cm_info.vlan = 0;
7011 in_params.mss = dev->ha->ifp->if_mtu - 40;
7013 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] "
7014 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d "
7016 NIPQUAD((in_params.cm_info.remote_ip[0])),
7017 NIPQUAD((in_params.cm_info.local_ip[0])),
7018 in_params.cm_info.remote_port, in_params.cm_info.local_port,
7019 in_params.cm_info.vlan);
7021 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);
7024 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n");
7028 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p"
7029 " private_data_len=%d rq_psn=%d\n",
7030 conn_param->ord, conn_param->ird, conn_param->private_data,
7031 conn_param->private_data_len, qp->rq_psn);
7033 in_params.cm_info.ord = conn_param->ord;
7034 in_params.cm_info.ird = conn_param->ird;
7035 in_params.cm_info.private_data = conn_param->private_data;
7036 in_params.cm_info.private_data_len = conn_param->private_data_len;
7037 in_params.qp = qp->ecore_qp;
7039 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
7041 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);
7044 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n");
7048 QL_DPRINT12(ha, "exit\n");
7053 cm_id->rem_ref(cm_id);
7056 QL_DPRINT12(ha, "exit [%d]\n", rc);
7061 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
7063 struct qlnxr_dev *dev;
7064 struct qlnxr_iw_listener *listener;
7065 struct ecore_iwarp_listen_in iparams;
7066 struct ecore_iwarp_listen_out oparams;
7067 struct sockaddr_in *laddr;
7071 dev = get_qlnxr_dev((cm_id->device));
7074 QL_DPRINT12(ha, "enter\n");
7076 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7079 laddr = (struct sockaddr_in *)&cm_id->local_addr;
7081 listener = kzalloc(sizeof(*listener), GFP_KERNEL);
7083 if (listener == NULL) {
7084 QL_DPRINT11(ha, "listener memory alloc failed\n");
7088 listener->dev = dev;
7089 cm_id->add_ref(cm_id);
7090 listener->cm_id = cm_id;
7091 listener->backlog = backlog;
7093 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in));
7094 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out));
7096 iparams.cb_context = listener;
7097 iparams.event_cb = qlnxr_iw_event_handler;
7098 iparams.max_backlog = backlog;
7100 iparams.ip_version = ECORE_TCP_IPV4;
7102 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);
7103 iparams.port = ntohs(laddr->sin_port);
7106 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n",
7107 NIPQUAD((laddr->sin_addr.s_addr)),
7108 laddr->sin_port, iparams.port);
7110 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);
7113 "ecore_iwarp_create_listen failed rc = %d\n", rc);
7117 listener->ecore_handle = oparams.handle;
7118 cm_id->provider_data = listener;
7120 QL_DPRINT12(ha, "exit\n");
7124 cm_id->rem_ref(cm_id);
7127 QL_DPRINT12(ha, "exit [%d]\n", rc);
7132 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
7134 struct qlnxr_iw_listener *listener = cm_id->provider_data;
7135 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));
7141 QL_DPRINT12(ha, "enter\n");
7143 if (listener->ecore_handle)
7144 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,
7145 listener->ecore_handle);
7147 cm_id->rem_ref(cm_id);
7149 QL_DPRINT12(ha, "exit [%d]\n", rc);
7154 qlnxr_iw_accept(struct iw_cm_id *cm_id,
7155 struct iw_cm_conn_param *conn_param)
7157 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7158 struct qlnxr_dev *dev = ep->dev;
7159 struct qlnxr_qp *qp;
7160 struct ecore_iwarp_accept_in params;
7166 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn);
7168 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7171 qp = idr_find(&dev->qpidr, conn_param->qpn);
7173 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n",
7179 cm_id->add_ref(cm_id);
7182 params.ep_context = ep->ecore_context;
7183 params.cb_context = ep;
7184 params.qp = ep->qp->ecore_qp;
7185 params.private_data = conn_param->private_data;
7186 params.private_data_len = conn_param->private_data_len;
7187 params.ird = conn_param->ird;
7188 params.ord = conn_param->ord;
7190 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms);
7192 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);
7196 QL_DPRINT12(ha, "exit\n");
7199 cm_id->rem_ref(cm_id);
7200 QL_DPRINT12(ha, "exit rc = %d\n", rc);
7205 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
7207 #if __FreeBSD_version >= 1102000
7209 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7210 struct qlnxr_dev *dev = ep->dev;
7211 struct ecore_iwarp_reject_in params;
7214 params.ep_context = ep->ecore_context;
7215 params.cb_context = ep;
7216 params.private_data = pdata;
7217 params.private_data_len = pdata_len;
7220 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms);
7226 printf("iWARP reject_cr not implemented\n");
7229 #endif /* #if __FreeBSD_version >= 1102000 */
7233 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp)
7235 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7240 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp);
7242 atomic_inc(&qp->refcnt);
7244 QL_DPRINT12(ha, "exit \n");
7249 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
7251 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7256 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp);
7258 if (atomic_dec_and_test(&qp->refcnt)) {
7259 qlnxr_idr_remove(qp->dev, qp->qp_id);
7263 QL_DPRINT12(ha, "exit \n");
7268 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn)
7270 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
7276 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn);
7278 qp = idr_find(&dev->qpidr, qpn);
7280 QL_DPRINT12(ha, "exit qp = %p\n", qp);