2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
4 * Copyright (c) 2004 Intel Corporation. All rights reserved.
5 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * $Id: verbs.c 1349 2004-12-16 21:09:43Z roland $
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/kernel.h>
47 #include <sys/libkern.h>
48 #include <sys/module.h>
49 #include <sys/endian.h>
51 #include <contrib/rdma/ib_verbs.h>
52 #include <contrib/rdma/ib_cache.h>
54 int ib_rate_to_mult(enum ib_rate rate)
57 case IB_RATE_2_5_GBPS: return 1;
58 case IB_RATE_5_GBPS: return 2;
59 case IB_RATE_10_GBPS: return 4;
60 case IB_RATE_20_GBPS: return 8;
61 case IB_RATE_30_GBPS: return 12;
62 case IB_RATE_40_GBPS: return 16;
63 case IB_RATE_60_GBPS: return 24;
64 case IB_RATE_80_GBPS: return 32;
65 case IB_RATE_120_GBPS: return 48;
70 enum ib_rate mult_to_ib_rate(int mult)
73 case 1: return IB_RATE_2_5_GBPS;
74 case 2: return IB_RATE_5_GBPS;
75 case 4: return IB_RATE_10_GBPS;
76 case 8: return IB_RATE_20_GBPS;
77 case 12: return IB_RATE_30_GBPS;
78 case 16: return IB_RATE_40_GBPS;
79 case 24: return IB_RATE_60_GBPS;
80 case 32: return IB_RATE_80_GBPS;
81 case 48: return IB_RATE_120_GBPS;
82 default: return IB_RATE_PORT_CURRENT;
86 enum rdma_transport_type
87 rdma_node_get_transport(enum rdma_node_type node_type)
91 case RDMA_NODE_IB_SWITCH:
92 case RDMA_NODE_IB_ROUTER:
93 return RDMA_TRANSPORT_IB;
95 return RDMA_TRANSPORT_IWARP;
97 panic("bad condition");
102 /* Protection domains */
104 struct ib_pd *ib_alloc_pd(struct ib_device *device)
108 pd = device->alloc_pd(device, NULL, NULL);
113 atomic_store_rel_int(&pd->usecnt, 0);
119 int ib_dealloc_pd(struct ib_pd *pd)
121 if (atomic_load_acq_int(&pd->usecnt))
124 return pd->device->dealloc_pd(pd);
127 /* Address handles */
129 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
133 ah = pd->device->create_ah(pd, ah_attr);
136 ah->device = pd->device;
139 atomic_add_acq_int(&pd->usecnt, 1);
145 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
146 struct ib_grh *grh, struct ib_ah_attr *ah_attr)
152 memset(ah_attr, 0, sizeof *ah_attr);
153 ah_attr->dlid = wc->slid;
154 ah_attr->sl = wc->sl;
155 ah_attr->src_path_bits = wc->dlid_path_bits;
156 ah_attr->port_num = port_num;
158 if (wc->wc_flags & IB_WC_GRH) {
159 ah_attr->ah_flags = IB_AH_GRH;
160 ah_attr->grh.dgid = grh->sgid;
162 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
167 ah_attr->grh.sgid_index = (u8) gid_index;
168 flow_class = be32toh(grh->version_tclass_flow);
169 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
170 ah_attr->grh.hop_limit = 0xFF;
171 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
176 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
177 struct ib_grh *grh, u8 port_num)
179 struct ib_ah_attr ah_attr;
182 ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
186 return ib_create_ah(pd, &ah_attr);
189 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
191 return ah->device->modify_ah ?
192 ah->device->modify_ah(ah, ah_attr) :
196 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
198 return ah->device->query_ah ?
199 ah->device->query_ah(ah, ah_attr) :
203 int ib_destroy_ah(struct ib_ah *ah)
209 ret = ah->device->destroy_ah(ah);
211 atomic_subtract_acq_int(&pd->usecnt, 1);
216 /* Shared receive queues */
218 struct ib_srq *ib_create_srq(struct ib_pd *pd,
219 struct ib_srq_init_attr *srq_init_attr)
223 if (!pd->device->create_srq)
224 return ERR_PTR(ENOSYS);
226 srq = pd->device->create_srq(pd, srq_init_attr, NULL);
229 srq->device = pd->device;
232 srq->event_handler = srq_init_attr->event_handler;
233 srq->srq_context = srq_init_attr->srq_context;
234 atomic_add_acq_int(&pd->usecnt, 1);
235 atomic_store_rel_int(&srq->usecnt, 0);
241 int ib_modify_srq(struct ib_srq *srq,
242 struct ib_srq_attr *srq_attr,
243 enum ib_srq_attr_mask srq_attr_mask)
245 return srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL);
248 int ib_query_srq(struct ib_srq *srq,
249 struct ib_srq_attr *srq_attr)
251 return srq->device->query_srq ?
252 srq->device->query_srq(srq, srq_attr) : ENOSYS;
255 int ib_destroy_srq(struct ib_srq *srq)
260 if (atomic_load_acq_int(&srq->usecnt))
265 ret = srq->device->destroy_srq(srq);
267 atomic_subtract_acq_int(&pd->usecnt, 1);
274 struct ib_qp *ib_create_qp(struct ib_pd *pd,
275 struct ib_qp_init_attr *qp_init_attr)
279 qp = pd->device->create_qp(pd, qp_init_attr, NULL);
282 qp->device = pd->device;
284 qp->send_cq = qp_init_attr->send_cq;
285 qp->recv_cq = qp_init_attr->recv_cq;
286 qp->srq = qp_init_attr->srq;
288 qp->event_handler = qp_init_attr->event_handler;
289 qp->qp_context = qp_init_attr->qp_context;
290 qp->qp_type = qp_init_attr->qp_type;
291 atomic_add_acq_int(&pd->usecnt, 1);
292 atomic_add_acq_int(&qp_init_attr->send_cq->usecnt, 1);
293 atomic_add_acq_int(&qp_init_attr->recv_cq->usecnt, 1);
294 if (qp_init_attr->srq)
295 atomic_add_acq_int(&qp_init_attr->srq->usecnt, 1);
301 static const struct {
303 enum ib_qp_attr_mask req_param[IB_QPT_RAW_ETY + 1];
304 enum ib_qp_attr_mask opt_param[IB_QPT_RAW_ETY + 1];
305 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
307 [IB_QPS_RESET] = { .valid = 1 },
308 [IB_QPS_ERR] = { .valid = 1 },
312 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
315 [IB_QPT_UC] = (IB_QP_PKEY_INDEX |
318 [IB_QPT_RC] = (IB_QP_PKEY_INDEX |
321 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
323 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
329 [IB_QPS_RESET] = { .valid = 1 },
330 [IB_QPS_ERR] = { .valid = 1 },
334 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
337 [IB_QPT_UC] = (IB_QP_PKEY_INDEX |
340 [IB_QPT_RC] = (IB_QP_PKEY_INDEX |
343 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
345 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
352 [IB_QPT_UC] = (IB_QP_AV |
356 [IB_QPT_RC] = (IB_QP_AV |
360 IB_QP_MAX_DEST_RD_ATOMIC |
361 IB_QP_MIN_RNR_TIMER),
364 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
366 [IB_QPT_UC] = (IB_QP_ALT_PATH |
369 [IB_QPT_RC] = (IB_QP_ALT_PATH |
372 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
374 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
380 [IB_QPS_RESET] = { .valid = 1 },
381 [IB_QPS_ERR] = { .valid = 1 },
385 [IB_QPT_UD] = IB_QP_SQ_PSN,
386 [IB_QPT_UC] = IB_QP_SQ_PSN,
387 [IB_QPT_RC] = (IB_QP_TIMEOUT |
391 IB_QP_MAX_QP_RD_ATOMIC),
392 [IB_QPT_SMI] = IB_QP_SQ_PSN,
393 [IB_QPT_GSI] = IB_QP_SQ_PSN,
396 [IB_QPT_UD] = (IB_QP_CUR_STATE |
398 [IB_QPT_UC] = (IB_QP_CUR_STATE |
401 IB_QP_PATH_MIG_STATE),
402 [IB_QPT_RC] = (IB_QP_CUR_STATE |
405 IB_QP_MIN_RNR_TIMER |
406 IB_QP_PATH_MIG_STATE),
407 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
409 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
415 [IB_QPS_RESET] = { .valid = 1 },
416 [IB_QPS_ERR] = { .valid = 1 },
420 [IB_QPT_UD] = (IB_QP_CUR_STATE |
422 [IB_QPT_UC] = (IB_QP_CUR_STATE |
425 IB_QP_PATH_MIG_STATE),
426 [IB_QPT_RC] = (IB_QP_CUR_STATE |
429 IB_QP_PATH_MIG_STATE |
430 IB_QP_MIN_RNR_TIMER),
431 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
433 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
440 [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY,
441 [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
442 [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
443 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
444 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
449 [IB_QPS_RESET] = { .valid = 1 },
450 [IB_QPS_ERR] = { .valid = 1 },
454 [IB_QPT_UD] = (IB_QP_CUR_STATE |
456 [IB_QPT_UC] = (IB_QP_CUR_STATE |
459 IB_QP_PATH_MIG_STATE),
460 [IB_QPT_RC] = (IB_QP_CUR_STATE |
463 IB_QP_MIN_RNR_TIMER |
464 IB_QP_PATH_MIG_STATE),
465 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
467 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
474 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
476 [IB_QPT_UC] = (IB_QP_AV |
480 IB_QP_PATH_MIG_STATE),
481 [IB_QPT_RC] = (IB_QP_PORT |
486 IB_QP_MAX_QP_RD_ATOMIC |
487 IB_QP_MAX_DEST_RD_ATOMIC |
491 IB_QP_MIN_RNR_TIMER |
492 IB_QP_PATH_MIG_STATE),
493 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
495 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
501 [IB_QPS_RESET] = { .valid = 1 },
502 [IB_QPS_ERR] = { .valid = 1 },
506 [IB_QPT_UD] = (IB_QP_CUR_STATE |
508 [IB_QPT_UC] = (IB_QP_CUR_STATE |
510 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
512 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
518 [IB_QPS_RESET] = { .valid = 1 },
519 [IB_QPS_ERR] = { .valid = 1 }
523 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
524 enum ib_qp_type type, enum ib_qp_attr_mask mask)
526 enum ib_qp_attr_mask req_param, opt_param;
528 if (cur_state < 0 || cur_state > IB_QPS_ERR ||
529 next_state < 0 || next_state > IB_QPS_ERR)
532 if (mask & IB_QP_CUR_STATE &&
533 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
534 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
537 if (!qp_state_table[cur_state][next_state].valid)
540 req_param = qp_state_table[cur_state][next_state].req_param[type];
541 opt_param = qp_state_table[cur_state][next_state].opt_param[type];
543 if ((mask & req_param) != req_param)
546 if (mask & ~(req_param | opt_param | IB_QP_STATE))
552 int ib_modify_qp(struct ib_qp *qp,
553 struct ib_qp_attr *qp_attr,
556 return qp->device->modify_qp(qp, qp_attr, qp_attr_mask, NULL);
559 int ib_query_qp(struct ib_qp *qp,
560 struct ib_qp_attr *qp_attr,
562 struct ib_qp_init_attr *qp_init_attr)
564 return qp->device->query_qp ?
565 qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) :
569 int ib_destroy_qp(struct ib_qp *qp)
572 struct ib_cq *scq, *rcq;
581 ret = qp->device->destroy_qp(qp);
583 atomic_subtract_acq_int(&pd->usecnt, 1);
584 atomic_subtract_acq_int(&scq->usecnt, 1);
585 atomic_subtract_acq_int(&rcq->usecnt, 1);
587 atomic_subtract_acq_int(&srq->usecnt, 1);
593 /* Completion queues */
595 struct ib_cq *ib_create_cq(struct ib_device *device,
596 ib_comp_handler comp_handler,
597 void (*event_handler)(struct ib_event *, void *),
598 void *cq_context, int cqe, int comp_vector)
602 cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
607 cq->comp_handler = comp_handler;
608 cq->event_handler = event_handler;
609 cq->cq_context = cq_context;
610 atomic_store_rel_int(&cq->usecnt, 0);
616 int ib_destroy_cq(struct ib_cq *cq)
618 if (atomic_load_acq_int(&cq->usecnt))
621 return cq->device->destroy_cq(cq);
624 int ib_resize_cq(struct ib_cq *cq, int cqe)
626 return cq->device->resize_cq ?
627 cq->device->resize_cq(cq, cqe, NULL) : ENOSYS;
632 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
636 mr = pd->device->get_dma_mr(pd, mr_access_flags);
639 mr->device = pd->device;
642 atomic_add_acq_int(&pd->usecnt, 1);
643 atomic_store_rel_int(&mr->usecnt, 0);
649 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
650 struct ib_phys_buf *phys_buf_array,
657 mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
658 mr_access_flags, iova_start);
661 mr->device = pd->device;
664 atomic_add_acq_int(&pd->usecnt, 1);
665 atomic_store_rel_int(&mr->usecnt, 0);
671 int ib_rereg_phys_mr(struct ib_mr *mr,
674 struct ib_phys_buf *phys_buf_array,
679 struct ib_pd *old_pd;
682 if (!mr->device->rereg_phys_mr)
685 if (atomic_load_acq_int(&mr->usecnt))
690 ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
691 phys_buf_array, num_phys_buf,
692 mr_access_flags, iova_start);
694 if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
695 atomic_subtract_acq_int(&old_pd->usecnt, 1);
696 atomic_add_acq_int(&pd->usecnt, 1);
702 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
704 return mr->device->query_mr ?
705 mr->device->query_mr(mr, mr_attr) : ENOSYS;
708 int ib_dereg_mr(struct ib_mr *mr)
713 if (atomic_load_acq_int(&mr->usecnt))
717 ret = mr->device->dereg_mr(mr);
719 atomic_subtract_acq_int(&pd->usecnt, 1);
726 struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
730 if (!pd->device->alloc_mw)
731 return ERR_PTR(ENOSYS);
733 mw = pd->device->alloc_mw(pd);
735 mw->device = pd->device;
738 atomic_add_acq_int(&pd->usecnt, 1);
744 int ib_dealloc_mw(struct ib_mw *mw)
750 ret = mw->device->dealloc_mw(mw);
752 atomic_subtract_acq_int(&pd->usecnt, 1);
757 /* "Fast" memory regions */
759 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
761 struct ib_fmr_attr *fmr_attr)
765 if (!pd->device->alloc_fmr)
766 return ERR_PTR(ENOSYS);
768 fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
770 fmr->device = pd->device;
772 atomic_add_acq_int(&pd->usecnt, 1);
778 int ib_unmap_fmr(struct ib_fmr_list_head *fmr_list)
782 if (TAILQ_EMPTY(fmr_list))
785 fmr = TAILQ_FIRST(fmr_list);
786 return fmr->device->unmap_fmr(fmr_list);
789 int ib_dealloc_fmr(struct ib_fmr *fmr)
795 ret = fmr->device->dealloc_fmr(fmr);
797 atomic_subtract_acq_int(&pd->usecnt, 1);
802 /* Multicast groups */
804 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
806 if (!qp->device->attach_mcast)
808 if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
811 return qp->device->attach_mcast(qp, gid, lid);
814 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
816 if (!qp->device->detach_mcast)
818 if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
821 return qp->device->detach_mcast(qp, gid, lid);