2 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
3 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #ifdef OSM_VENDOR_INTF_MTL
37 /* - Mellanox Confidential and Proprietary -
39 * Copyright (C) Jul. 2001, Mellanox Technologies Ltd. ALL RIGHTS RESERVED.
41 * Except as specifically permitted herein, no portion of the information,
42 * including but not limited to object code and source code, may be reproduced,
43 * modified, distributed, republished or otherwise exploited in any form or by
44 * any means for any purpose without the prior written permission of Mellanox
45 * Technologies Ltd. Use of software subject to the terms and conditions
46 * detailed in the file "LICENSE.txt".
48 * End of legal section ......................................................
50 * osmt_mtl_regular_qp.c -
51 * Provide Simple Interface for Sending and Receiving MADS through a regular QP
69 #include <mtl_common.h>
72 #include <vapi_common.h>
74 #include <osmt_mtl_regular_qp.h>
75 #include <complib/cl_types.h>
77 * Initialize the QP etc.
78 * Given in res: port_num, max_outs_sq, max_outs_rq
80 VAPI_ret_t osmt_mtl_get_qp_resources(IN OUT osmt_mtl_mad_res_t * res)
83 VAPI_hca_port_t hca_port_info;
84 VAPI_qp_init_attr_t qp_init_attr;
85 VAPI_qp_prop_t qp_prop;
86 VAPI_cqe_num_t act_num;
90 VAPI_query_hca_port_prop(res->hca_hndl, res->port_num,
93 res->slid = hca_port_info.lid;
96 ret = VAPI_alloc_pd(res->hca_hndl, &(res->pd_hndl));
99 /* Create CQ for RQ and SQ *//* TBD - Check we have enough act nums */
101 VAPI_create_cq(res->hca_hndl, res->max_outs_sq + 1,
102 &(res->sq_cq_hndl), &act_num);
105 VAPI_create_cq(res->hca_hndl, res->max_outs_rq + 1,
106 &(res->rq_cq_hndl), &act_num);
109 /* register event handlers for polling(block mode) internal use */
110 /* ret= EVAPI_set_comp_eventh(res->hca_hndl,res->rq_cq_hndl, */
111 /* EVAPI_POLL_CQ_UNBLOCK_HANDLER,NULL,&(res->rq_cq_eventh)); */
112 /* VAPI_CHECK_RET; */
113 /* ret= EVAPI_set_comp_eventh(res->hca_hndl,res->sq_cq_hndl, */
114 /* EVAPI_POLL_CQ_UNBLOCK_HANDLER,NULL,&(res->sq_cq_eventh)); */
115 /* VAPI_CHECK_RET; */
118 qp_init_attr.cap.max_oust_wr_sq = res->max_outs_sq + 1;
119 qp_init_attr.cap.max_oust_wr_rq = res->max_outs_rq + 1;
120 qp_init_attr.cap.max_sg_size_sq = 4;
121 qp_init_attr.cap.max_sg_size_rq = 4;
123 qp_init_attr.pd_hndl = res->pd_hndl;
124 qp_init_attr.rdd_hndl = 0;
125 qp_init_attr.rq_cq_hndl = res->rq_cq_hndl;
126 qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR; /* That's default for IB */
127 qp_init_attr.sq_cq_hndl = res->sq_cq_hndl;
128 qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;
129 qp_init_attr.ts_type = VAPI_TS_UD;
132 VAPI_create_qp(res->hca_hndl, &qp_init_attr, &(res->qp_hndl),
135 res->qp_id.qp_num = qp_prop.qp_num;
140 VAPI_ret_t osmt_mtl_qp_init(osmt_mtl_mad_res_t * res)
144 VAPI_qp_attr_t qp_attr;
145 VAPI_qp_attr_mask_t qp_attr_mask;
146 VAPI_qp_cap_t qp_cap;
152 QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
153 qp_attr.qp_state = VAPI_INIT;
154 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
156 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
157 qp_attr.port = res->port_num;
158 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
159 qp_attr.qkey = res->qkey;
160 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QKEY);
162 /* If I do not set this mask, I get an error from HH. QPM should catch it */
164 VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,
172 VAPI_ret_t osmt_mtl_qp_2_rtr_rts(osmt_mtl_mad_res_t * res)
176 VAPI_qp_attr_t qp_attr;
177 VAPI_qp_attr_mask_t qp_attr_mask;
178 VAPI_qp_cap_t qp_cap;
184 QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
185 qp_attr.qp_state = VAPI_RTR;
186 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
187 /* qp_attr.rq_psn = 0; */
188 /* QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RQ_PSN); */
191 VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,
199 QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
200 qp_attr.qp_state = VAPI_RTS;
201 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
203 QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
206 VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,
213 VAPI_ret_t osmt_mtl_mad_create_mr(osmt_mtl_mad_res_t * res)
218 VAPI_mrw_t mr_in, mr_out;
221 (MAD_SIZE + GRH_LEN) * (res->max_outs_sq + res->max_outs_rq + 1);
223 /* Register single memory address region for all buffers */
224 res->buf_ptr = VMALLOC(res->buf_size);
226 if (res->buf_ptr == ((VAPI_virt_addr_t) NULL)) {
231 /* Enable local and remote access to memory region */
232 mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
234 mr_in.pd_hndl = res->pd_hndl;
236 mr_in.size = res->buf_size;
237 ASSERT_VOIDP2UINTN(res->buf_ptr);
238 mr_in.start = (VAPI_virt_addr_t) (uintn_t) (res->buf_ptr);
239 mr_in.type = VAPI_MR;
241 ret = VAPI_register_mr(res->hca_hndl, &mr_in, &(res->mr_hndl), &mr_out);
244 res->l_key = mr_out.l_key;
249 VAPI_ret_t osmt_mtl_init_opened_hca(osmt_mtl_mad_res_t * res)
253 res->pd_hndl = VAPI_INVAL_HNDL;
254 res->rq_cq_hndl = VAPI_INVAL_HNDL;
255 res->sq_cq_hndl = VAPI_INVAL_HNDL;
256 res->sq_cq_eventh = VAPI_INVAL_HNDL;
257 res->rq_cq_eventh = VAPI_INVAL_HNDL;
258 res->qp_hndl = VAPI_INVAL_HNDL;
259 res->mr_hndl = VAPI_INVAL_HNDL;
265 ret = osmt_mtl_get_qp_resources(res);
266 if (ret != VAPI_OK) {
274 ret = osmt_mtl_qp_init(res);
275 if (ret != VAPI_OK) {
280 * Initialize memory regions
283 ret = osmt_mtl_mad_create_mr(res);
284 if (ret != VAPI_OK) {
288 /* only now move to RTR and RTS */
289 ret = osmt_mtl_qp_2_rtr_rts(res);
290 if (ret != VAPI_OK) {
297 VAPI_ret_t osmt_mtl_mad_cleanup(osmt_mtl_mad_res_t * res)
299 if (res->qp_hndl != VAPI_INVAL_HNDL) {
300 VAPI_destroy_qp(res->hca_hndl, res->qp_hndl);
302 if (res->sq_cq_eventh != VAPI_INVAL_HNDL) {
303 EVAPI_clear_comp_eventh(res->hca_hndl, res->sq_cq_eventh);
305 if (res->rq_cq_eventh != VAPI_INVAL_HNDL) {
306 EVAPI_clear_comp_eventh(res->hca_hndl, res->rq_cq_eventh);
308 if (res->rq_cq_hndl != VAPI_INVAL_HNDL) {
309 VAPI_destroy_cq(res->hca_hndl, res->rq_cq_hndl);
311 if (res->sq_cq_hndl != VAPI_INVAL_HNDL) {
312 VAPI_destroy_cq(res->hca_hndl, res->sq_cq_hndl);
314 if (res->mr_hndl != VAPI_INVAL_HNDL) {
315 VAPI_deregister_mr(res->hca_hndl, res->mr_hndl);
317 if (res->pd_hndl != VAPI_INVAL_HNDL) {
318 VAPI_dealloc_pd(res->hca_hndl, res->pd_hndl);
321 /* open/close of HCA should be done system wide - not per application */
322 if (res->hca_hndl != VAPI_INVAL_HNDL) {
323 VAPI_close_hca(res->hca_hndl); /* TBD: HCA_open/close should be done on a system wide basis */
329 VAPI_ret_t osmt_mtl_create_av(osmt_mtl_mad_res_t * res, int16_t dlid,
330 VAPI_ud_av_hndl_t * avh_p)
336 av.port = res->port_num;
337 av.sl = 0; /* dest->sl; */
338 av.src_path_bits = 0; /* dest->ee_dlid.dst_path_bits; */
343 ret = VAPI_create_addr_hndl(res->hca_hndl, res->pd_hndl, &av, avh_p);
344 if (ret != VAPI_OK) {
345 MTL_ERROR1("%s: failed VAPI_create_addr_hndl (%s)\n", __func__,
346 VAPI_strerror_sym(ret));
352 VAPI_ret_t osmt_mtl_mad_send(osmt_mtl_mad_res_t * res, VAPI_wr_id_t id,
353 void *mad, VAPI_qp_num_t dest_qp, IB_sl_t sl,
354 u_int32_t dest_qkey, VAPI_ud_av_hndl_t avh)
357 VAPI_sg_lst_entry_t sg_entry;
360 /* building SEND request */
361 sr.opcode = VAPI_SEND;
363 sr.remote_qp = dest_qp;
364 sr.remote_qkey = dest_qkey;
369 sr.comp_type = VAPI_SIGNALED;
371 sr.sg_lst_p = &sg_entry;
372 ASSERT_VOIDP2UINTN(mad);
373 sg_entry.addr = (VAPI_virt_addr_t) (uintn_t) (mad);
374 sg_entry.len = MAD_SIZE;
375 sg_entry.lkey = res->l_key;
377 ret = VAPI_post_sr(res->hca_hndl, res->qp_hndl, &sr);
378 if (ret != VAPI_OK) {
379 MTL_ERROR1(__FUNCTION__ ": failed VAPI_post_sr (%s)\n",
380 VAPI_strerror_sym(ret));
387 int osmt_mtl_mad_post_recv_bufs(osmt_mtl_mad_res_t * res, void *buf_array,
388 u_int32_t num_o_bufs, u_int32_t size,
389 VAPI_wr_id_t start_id)
394 VAPI_sg_lst_entry_t sg_entry;
397 rr.opcode = VAPI_RECEIVE;
398 rr.comp_type = VAPI_SIGNALED; /* All with CQE (IB compliant) */
399 rr.sg_lst_len = 1; /* single buffers */
400 rr.sg_lst_p = &sg_entry;
401 sg_entry.lkey = res->l_key;
403 for (i = 0; i < num_o_bufs; i++) {
404 rr.id = start_id + i; /* WQE id used is the index to buffers ptr array */
405 ASSERT_VOIDP2UINTN(cur_buf);
406 sg_entry.addr = (VAPI_virt_addr_t) (uintn_t) cur_buf;
408 memset(cur_buf, 0x00, size); /* fill with 0 */
409 ret = VAPI_post_rr(res->hca_hndl, res->qp_hndl, &rr);
410 if (ret != VAPI_OK) {
411 MTL_ERROR1(__FUNCTION__
412 ": failed posting RQ WQE (%s)\n",
413 VAPI_strerror_sym(ret));
416 MTL_DEBUG4(__FUNCTION__ ": posted buf at %p\n", cur_buf);
420 return i; /* num of buffers posted */
423 VAPI_ret_t osmt_mtl_mad_poll4cqe(VAPI_hca_hndl_t hca, VAPI_cq_hndl_t cq,
424 VAPI_wc_desc_t * wc_desc_p,
425 u_int32_t max_poll, u_int32_t poll_sleep,
426 VAPI_ud_av_hndl_t * avh_p)
428 VAPI_ret_t ret = VAPI_CQ_EMPTY;
429 u_int32_t poll_cnt = 0;
431 /* wait for something to arrive */
432 while ((ret == VAPI_CQ_EMPTY) && (poll_cnt < max_poll)) {
433 ret = VAPI_poll_cq(hca, cq, wc_desc_p);
434 /* don't sleep if we already succeeded) */
435 if (ret != VAPI_CQ_EMPTY) {
442 /* if passed an AVH to destory - do it */
444 VAPI_destroy_addr_hndl(hca, *avh_p);
447 if ((poll_cnt == max_poll) && (ret == VAPI_CQ_EMPTY)) {
448 MTL_DEBUG1(__FUNCTION__
449 ": Failed to get completion on wq after %d polls.\n",
451 return VAPI_CQ_EMPTY;
454 if (ret != VAPI_OK) {
455 MTL_DEBUG1(__FUNCTION__
456 ": VAPI_poll_cq failed with ret=%s on sq_cq\n",
457 mtl_strerror_sym(ret));
461 if (wc_desc_p->status != VAPI_SUCCESS) {
462 MTL_DEBUG1(__FUNCTION__ ": completion error (%d) detected\n",
469 #endif /* OSM_VENDOR_INTF_MTL */