2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #endif /* HAVE_CONFIG_H */
43 #include <netinet/in.h>
46 #include "mthca-abi.h"
48 int mthca_query_device(struct ibv_context *context, struct ibv_device_attr *attr)
50 struct ibv_query_device cmd;
52 unsigned major, minor, sub_minor;
55 ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd);
59 major = (raw_fw_ver >> 32) & 0xffff;
60 minor = (raw_fw_ver >> 16) & 0xffff;
61 sub_minor = raw_fw_ver & 0xffff;
63 snprintf(attr->fw_ver, sizeof attr->fw_ver,
64 "%d.%d.%d", major, minor, sub_minor);
69 int mthca_query_port(struct ibv_context *context, uint8_t port,
70 struct ibv_port_attr *attr)
72 struct ibv_query_port cmd;
74 return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
77 struct ibv_pd *mthca_alloc_pd(struct ibv_context *context)
79 struct ibv_alloc_pd cmd;
80 struct mthca_alloc_pd_resp resp;
83 pd = malloc(sizeof *pd);
87 if (!mthca_is_memfree(context)) {
89 if (pthread_mutex_init(&pd->ah_mutex, NULL)) {
95 if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd,
96 &resp.ibv_resp, sizeof resp)) {
106 int mthca_free_pd(struct ibv_pd *pd)
110 ret = ibv_cmd_dealloc_pd(pd);
118 static struct ibv_mr *__mthca_reg_mr(struct ibv_pd *pd, void *addr,
119 size_t length, uint64_t hca_va,
120 enum ibv_access_flags access,
124 struct mthca_reg_mr cmd;
128 * Old kernels just ignore the extra data we pass in with the
129 * reg_mr command structure, so there's no need to add an ABI
130 * version check here (and indeed the kernel ABI was not
131 * incremented due to this change).
133 cmd.mr_attrs = dma_sync ? MTHCA_MR_DMASYNC : 0;
136 mr = malloc(sizeof *mr);
140 #ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS
142 struct ibv_reg_mr_resp resp;
144 ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, mr,
145 &cmd.ibv_cmd, sizeof cmd, &resp, sizeof resp);
148 ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, mr,
149 &cmd.ibv_cmd, sizeof cmd);
159 struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr,
160 size_t length, enum ibv_access_flags access)
162 return __mthca_reg_mr(pd, addr, length, (uintptr_t) addr, access, 0);
165 int mthca_dereg_mr(struct ibv_mr *mr)
169 ret = ibv_cmd_dereg_mr(mr);
177 static int align_cq_size(int cqe)
181 for (nent = 1; nent <= cqe; nent <<= 1)
187 struct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe,
188 struct ibv_comp_channel *channel,
191 struct mthca_create_cq cmd;
192 struct mthca_create_cq_resp resp;
196 /* Sanity check CQ size before proceeding */
200 cq = malloc(sizeof *cq);
206 if (pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE))
209 cqe = align_cq_size(cqe);
210 if (mthca_alloc_cq_buf(to_mdev(context->device), &cq->buf, cqe))
213 cq->mr = __mthca_reg_mr(to_mctx(context)->pd, cq->buf.buf,
214 cqe * MTHCA_CQ_ENTRY_SIZE,
215 0, IBV_ACCESS_LOCAL_WRITE, 1);
219 cq->mr->context = context;
221 if (mthca_is_memfree(context)) {
223 cq->set_ci_db_index = mthca_alloc_db(to_mctx(context)->db_tab,
224 MTHCA_DB_TYPE_CQ_SET_CI,
226 if (cq->set_ci_db_index < 0)
229 cq->arm_db_index = mthca_alloc_db(to_mctx(context)->db_tab,
230 MTHCA_DB_TYPE_CQ_ARM,
232 if (cq->arm_db_index < 0)
235 cmd.arm_db_page = db_align(cq->arm_db);
236 cmd.set_db_page = db_align(cq->set_ci_db);
237 cmd.arm_db_index = cq->arm_db_index;
238 cmd.set_db_index = cq->set_ci_db_index;
240 cmd.arm_db_page = cmd.set_db_page =
241 cmd.arm_db_index = cmd.set_db_index = 0;
244 cmd.lkey = cq->mr->lkey;
245 cmd.pdn = to_mpd(to_mctx(context)->pd)->pdn;
246 ret = ibv_cmd_create_cq(context, cqe - 1, channel, comp_vector,
247 &cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
248 &resp.ibv_resp, sizeof resp);
254 if (mthca_is_memfree(context)) {
255 mthca_set_db_qn(cq->set_ci_db, MTHCA_DB_TYPE_CQ_SET_CI, cq->cqn);
256 mthca_set_db_qn(cq->arm_db, MTHCA_DB_TYPE_CQ_ARM, cq->cqn);
262 if (mthca_is_memfree(context))
263 mthca_free_db(to_mctx(context)->db_tab, MTHCA_DB_TYPE_CQ_ARM,
267 if (mthca_is_memfree(context))
268 mthca_free_db(to_mctx(context)->db_tab, MTHCA_DB_TYPE_CQ_SET_CI,
269 cq->set_ci_db_index);
272 mthca_dereg_mr(cq->mr);
275 mthca_free_buf(&cq->buf);
283 int mthca_resize_cq(struct ibv_cq *ibcq, int cqe)
285 struct mthca_cq *cq = to_mcq(ibcq);
286 struct mthca_resize_cq cmd;
288 struct mthca_buf buf;
292 /* Sanity check CQ size before proceeding */
296 pthread_spin_lock(&cq->lock);
298 cqe = align_cq_size(cqe);
299 if (cqe == ibcq->cqe + 1) {
304 ret = mthca_alloc_cq_buf(to_mdev(ibcq->context->device), &buf, cqe);
308 mr = __mthca_reg_mr(to_mctx(ibcq->context)->pd, buf.buf,
309 cqe * MTHCA_CQ_ENTRY_SIZE,
310 0, IBV_ACCESS_LOCAL_WRITE, 1);
312 mthca_free_buf(&buf);
317 mr->context = ibcq->context;
322 #ifdef IBV_CMD_RESIZE_CQ_HAS_RESP_PARAMS
324 struct ibv_resize_cq_resp resp;
325 ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd,
329 ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd);
333 mthca_free_buf(&buf);
337 mthca_cq_resize_copy_cqes(cq, buf.buf, old_cqe);
339 mthca_dereg_mr(cq->mr);
340 mthca_free_buf(&cq->buf);
346 pthread_spin_unlock(&cq->lock);
350 int mthca_destroy_cq(struct ibv_cq *cq)
354 ret = ibv_cmd_destroy_cq(cq);
358 if (mthca_is_memfree(cq->context)) {
359 mthca_free_db(to_mctx(cq->context)->db_tab, MTHCA_DB_TYPE_CQ_SET_CI,
360 to_mcq(cq)->set_ci_db_index);
361 mthca_free_db(to_mctx(cq->context)->db_tab, MTHCA_DB_TYPE_CQ_ARM,
362 to_mcq(cq)->arm_db_index);
365 mthca_dereg_mr(to_mcq(cq)->mr);
366 mthca_free_buf(&to_mcq(cq)->buf);
372 static int align_queue_size(struct ibv_context *context, int size, int spare)
377 * If someone asks for a 0-sized queue, presumably they're not
378 * going to use it. So don't mess with their size.
383 if (mthca_is_memfree(context)) {
384 for (ret = 1; ret < size + spare; ret <<= 1)
392 struct ibv_srq *mthca_create_srq(struct ibv_pd *pd,
393 struct ibv_srq_init_attr *attr)
395 struct mthca_create_srq cmd;
396 struct mthca_create_srq_resp resp;
397 struct mthca_srq *srq;
400 /* Sanity check SRQ size before proceeding */
401 if (attr->attr.max_wr > 1 << 16 || attr->attr.max_sge > 64)
404 srq = malloc(sizeof *srq);
408 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
411 srq->max = align_queue_size(pd->context, attr->attr.max_wr, 1);
412 srq->max_gs = attr->attr.max_sge;
415 if (mthca_alloc_srq_buf(pd, &attr->attr, srq))
418 srq->mr = __mthca_reg_mr(pd, srq->buf.buf, srq->buf_size, 0, 0, 0);
422 srq->mr->context = pd->context;
424 if (mthca_is_memfree(pd->context)) {
425 srq->db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab,
426 MTHCA_DB_TYPE_SRQ, &srq->db);
427 if (srq->db_index < 0)
430 cmd.db_page = db_align(srq->db);
431 cmd.db_index = srq->db_index;
433 cmd.db_page = cmd.db_index = 0;
436 cmd.lkey = srq->mr->lkey;
438 ret = ibv_cmd_create_srq(pd, &srq->ibv_srq, attr,
439 &cmd.ibv_cmd, sizeof cmd,
440 &resp.ibv_resp, sizeof resp);
444 srq->srqn = resp.srqn;
446 if (mthca_is_memfree(pd->context))
447 mthca_set_db_qn(srq->db, MTHCA_DB_TYPE_SRQ, srq->srqn);
449 return &srq->ibv_srq;
452 if (mthca_is_memfree(pd->context))
453 mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_SRQ,
457 mthca_dereg_mr(srq->mr);
461 mthca_free_buf(&srq->buf);
469 int mthca_modify_srq(struct ibv_srq *srq,
470 struct ibv_srq_attr *attr,
471 enum ibv_srq_attr_mask attr_mask)
473 struct ibv_modify_srq cmd;
475 return ibv_cmd_modify_srq(srq, attr, attr_mask, &cmd, sizeof cmd);
478 int mthca_query_srq(struct ibv_srq *srq,
479 struct ibv_srq_attr *attr)
481 struct ibv_query_srq cmd;
483 return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd);
486 int mthca_destroy_srq(struct ibv_srq *srq)
490 ret = ibv_cmd_destroy_srq(srq);
494 if (mthca_is_memfree(srq->context))
495 mthca_free_db(to_mctx(srq->context)->db_tab, MTHCA_DB_TYPE_SRQ,
496 to_msrq(srq)->db_index);
498 mthca_dereg_mr(to_msrq(srq)->mr);
500 mthca_free_buf(&to_msrq(srq)->buf);
501 free(to_msrq(srq)->wrid);
507 struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
509 struct mthca_create_qp cmd;
510 struct ibv_create_qp_resp resp;
514 /* Sanity check QP size before proceeding */
515 if (attr->cap.max_send_wr > 65536 ||
516 attr->cap.max_recv_wr > 65536 ||
517 attr->cap.max_send_sge > 64 ||
518 attr->cap.max_recv_sge > 64 ||
519 attr->cap.max_inline_data > 1024)
522 qp = malloc(sizeof *qp);
526 qp->sq.max = align_queue_size(pd->context, attr->cap.max_send_wr, 0);
527 qp->rq.max = align_queue_size(pd->context, attr->cap.max_recv_wr, 0);
529 if (mthca_alloc_qp_buf(pd, &attr->cap, attr->qp_type, qp))
532 mthca_init_qp_indices(qp);
534 if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) ||
535 pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
538 qp->mr = __mthca_reg_mr(pd, qp->buf.buf, qp->buf_size, 0, 0, 0);
542 qp->mr->context = pd->context;
544 cmd.lkey = qp->mr->lkey;
547 if (mthca_is_memfree(pd->context)) {
548 qp->sq.db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab,
551 if (qp->sq.db_index < 0)
554 qp->rq.db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab,
557 if (qp->rq.db_index < 0)
560 cmd.sq_db_page = db_align(qp->sq.db);
561 cmd.rq_db_page = db_align(qp->rq.db);
562 cmd.sq_db_index = qp->sq.db_index;
563 cmd.rq_db_index = qp->rq.db_index;
565 cmd.sq_db_page = cmd.rq_db_page =
566 cmd.sq_db_index = cmd.rq_db_index = 0;
569 pthread_mutex_lock(&to_mctx(pd->context)->qp_table_mutex);
570 ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
575 if (mthca_is_memfree(pd->context)) {
576 mthca_set_db_qn(qp->sq.db, MTHCA_DB_TYPE_SQ, qp->ibv_qp.qp_num);
577 mthca_set_db_qn(qp->rq.db, MTHCA_DB_TYPE_RQ, qp->ibv_qp.qp_num);
580 ret = mthca_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
583 pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex);
585 qp->sq.max = attr->cap.max_send_wr;
586 qp->rq.max = attr->cap.max_recv_wr;
587 qp->sq.max_gs = attr->cap.max_send_sge;
588 qp->rq.max_gs = attr->cap.max_recv_sge;
589 qp->max_inline_data = attr->cap.max_inline_data;
594 ibv_cmd_destroy_qp(&qp->ibv_qp);
597 pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex);
598 if (mthca_is_memfree(pd->context))
599 mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_RQ,
603 if (mthca_is_memfree(pd->context))
604 mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_SQ,
608 mthca_dereg_mr(qp->mr);
612 mthca_free_buf(&qp->buf);
620 int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
621 enum ibv_qp_attr_mask attr_mask,
622 struct ibv_qp_init_attr *init_attr)
624 struct ibv_query_qp cmd;
626 return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof cmd);
629 int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
630 enum ibv_qp_attr_mask attr_mask)
632 struct ibv_modify_qp cmd;
635 ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
638 (attr_mask & IBV_QP_STATE) &&
639 attr->qp_state == IBV_QPS_RESET) {
640 mthca_cq_clean(to_mcq(qp->recv_cq), qp->qp_num,
641 qp->srq ? to_msrq(qp->srq) : NULL);
642 if (qp->send_cq != qp->recv_cq)
643 mthca_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL);
645 mthca_init_qp_indices(to_mqp(qp));
647 if (mthca_is_memfree(qp->context)) {
648 *to_mqp(qp)->sq.db = 0;
649 *to_mqp(qp)->rq.db = 0;
656 static void mthca_lock_cqs(struct ibv_qp *qp)
658 struct mthca_cq *send_cq = to_mcq(qp->send_cq);
659 struct mthca_cq *recv_cq = to_mcq(qp->recv_cq);
661 if (send_cq == recv_cq)
662 pthread_spin_lock(&send_cq->lock);
663 else if (send_cq->cqn < recv_cq->cqn) {
664 pthread_spin_lock(&send_cq->lock);
665 pthread_spin_lock(&recv_cq->lock);
667 pthread_spin_lock(&recv_cq->lock);
668 pthread_spin_lock(&send_cq->lock);
672 static void mthca_unlock_cqs(struct ibv_qp *qp)
674 struct mthca_cq *send_cq = to_mcq(qp->send_cq);
675 struct mthca_cq *recv_cq = to_mcq(qp->recv_cq);
677 if (send_cq == recv_cq)
678 pthread_spin_unlock(&send_cq->lock);
679 else if (send_cq->cqn < recv_cq->cqn) {
680 pthread_spin_unlock(&recv_cq->lock);
681 pthread_spin_unlock(&send_cq->lock);
683 pthread_spin_unlock(&send_cq->lock);
684 pthread_spin_unlock(&recv_cq->lock);
688 int mthca_destroy_qp(struct ibv_qp *qp)
692 pthread_mutex_lock(&to_mctx(qp->context)->qp_table_mutex);
693 ret = ibv_cmd_destroy_qp(qp);
695 pthread_mutex_unlock(&to_mctx(qp->context)->qp_table_mutex);
701 __mthca_cq_clean(to_mcq(qp->recv_cq), qp->qp_num,
702 qp->srq ? to_msrq(qp->srq) : NULL);
703 if (qp->send_cq != qp->recv_cq)
704 __mthca_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL);
706 mthca_clear_qp(to_mctx(qp->context), qp->qp_num);
708 mthca_unlock_cqs(qp);
709 pthread_mutex_unlock(&to_mctx(qp->context)->qp_table_mutex);
711 if (mthca_is_memfree(qp->context)) {
712 mthca_free_db(to_mctx(qp->context)->db_tab, MTHCA_DB_TYPE_RQ,
713 to_mqp(qp)->rq.db_index);
714 mthca_free_db(to_mctx(qp->context)->db_tab, MTHCA_DB_TYPE_SQ,
715 to_mqp(qp)->sq.db_index);
718 mthca_dereg_mr(to_mqp(qp)->mr);
719 mthca_free_buf(&to_mqp(qp)->buf);
720 free(to_mqp(qp)->wrid);
726 struct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
730 ah = malloc(sizeof *ah);
734 if (mthca_alloc_av(to_mpd(pd), attr, ah)) {
742 int mthca_destroy_ah(struct ibv_ah *ah)
744 mthca_free_av(to_mah(ah));
750 int mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
752 return ibv_cmd_attach_mcast(qp, gid, lid);
755 int mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
757 return ibv_cmd_detach_mcast(qp, gid, lid);