2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 #endif /* HAVE_CONFIG_H */
40 #ifdef OSM_VENDOR_INTF_MTL
44 #include <opensm/osm_helper.h>
45 #include <opensm/osm_log.h>
46 /* HACK - I do not know how to prevent complib from loading kernel H files */
48 #include <vendor/osm_vendor_mtl.h>
49 #include <vendor/osm_vendor_api.h>
50 #include <opensm/osm_subnet.h>
51 #include <opensm/osm_opensm.h>
52 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
53 #include <vendor/osm_mtl_bind.h>
56 Since a race can accure on requests. Meaning - a response is received before
57 the send_callback is called - we will save both the madw_p and the fact
58 whether or not it is a response. A race can occure only on requests that did
59 not fail, and then the madw_p will be put back in the pool before the callback.
61 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
65 CL_ASSERT(p_madw->p_mad);
67 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
69 ib_mad_is_response(p_madw->p_mad);
74 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
75 OUT uint8_t * is_resp,
76 OUT osm_madw_t ** pp_madw)
78 *is_resp = wrid & 0x0000000000000001;
80 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
83 /**********************************************************************
84 * IB_MGT to OSM ADDRESS VECTOR
85 **********************************************************************/
87 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
88 IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
90 OUT osm_mad_addr_t * p_mad_addr)
92 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
93 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
94 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
95 p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
98 p_mad_addr->addr_type.smi.source_lid =
99 cl_hton16(p_rcv_desc->remote_lid);
100 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */
103 /* seems to me there is a IBMGT bug reversing the QPN ... */
104 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
105 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
107 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
108 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
109 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
110 /* the full PKey table - than go by the index. */
111 /* since this does not seem reasonable to me I simply use the default */
112 /* There is a TAVOR limitation that only one P_KEY is supported per */
113 /* QP - so QP1 must use IB_DEFAULT_PKEY */
114 p_mad_addr->addr_type.gsi.pkey_ix = 0;
115 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
117 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
118 /* copy the GRH data if relevant */
119 if (p_mad_addr->addr_type.gsi.global_route) {
120 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
121 ib_grh_set_ver_class_flow(p_rcv_desc->grh.
127 p_mad_addr->addr_type.gsi.grh_info.hop_limit =
128 p_rcv_desc->grh.hop_limit;
129 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
130 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
131 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
132 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
137 /**********************************************************************
138 * OSM ADDR VECTOR TO IB_MGT
139 **********************************************************************/
141 __osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,
142 IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
145 /* For global destination or Multicast address: */
148 memset(p_av, 0, sizeof(IB_ud_av_t));
150 p_av->src_path_bits = p_mad_addr->path_bits;
151 p_av->static_rate = p_mad_addr->static_rate;
152 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
155 p_av->sl = 0; /* Just to note we use 0 here. */
157 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
158 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
160 if (p_mad_addr->addr_type.gsi.global_route) {
161 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
162 grh_info.ver_class_flow, &ver,
163 &p_av->traffic_class,
166 p_mad_addr->addr_type.gsi.grh_info.hop_limit;
167 p_av->sgid_index = 0; /* we always use source GID 0 */
169 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
176 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
178 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
179 osm_vendor_t *p_vend = p_bind->p_vend;
181 VAPI_hca_attr_t attr_mod;
182 VAPI_hca_attr_mask_t attr_mask;
184 OSM_LOG_ENTER(p_vend->p_log);
186 memset(&attr_mod, 0, sizeof(attr_mod));
187 memset(&attr_mask, 0, sizeof(attr_mask));
189 attr_mod.is_sm = FALSE;
190 attr_mask = HCA_ATTR_IS_SM;
193 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
195 if (status != VAPI_OK) {
196 osm_log(p_vend->p_log, OSM_LOG_ERROR,
197 "__osm_vendor_clear_sm: ERR 3C21: "
198 "Unable set 'IS_SM' bit in port attributes (%d).\n",
202 OSM_LOG_EXIT(p_vend->p_log);
205 /**********************************************************************
206 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
207 **********************************************************************/
208 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
210 memset(p_vend, 0, sizeof(*p_vend));
213 /**********************************************************************
214 * DEALOCATE osm_vendor_t
215 **********************************************************************/
216 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
218 osm_vendor_mgt_bind_t *vendor_mgt_bind_p;
219 IB_MGT_ret_t mgt_ret;
220 OSM_LOG_ENTER(p_vend->p_log);
222 if (p_vend->h_al != NULL) {
223 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
224 if (vendor_mgt_bind_p->gsi_init) {
226 /* un register the class */
227 /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */
229 IB_MGT_unbind_gsi_class(vendor_mgt_bind_p->
232 if (mgt_ret != IB_MGT_OK) {
233 osm_log(p_vend->p_log, OSM_LOG_ERROR,
234 "osm_vendor_destroy: ERR 3C03: "
235 "Fail to unbind the SA class.\n");
238 /* un bind the handle */
239 if (IB_MGT_release_handle
240 (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) {
241 osm_log(p_vend->p_log, OSM_LOG_ERROR,
242 "osm_vendor_destroy: ERR 3C02: "
243 "Fail to unbind the SA GSI handle.\n");
245 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
246 "osm_vendor_destroy: DBG 1002: "
247 "Unbind the GSI handles.\n");
249 if (vendor_mgt_bind_p->smi_init) {
250 /* first - clear the IS_SM in the capability mask */
251 __osm_vendor_clear_sm((osm_bind_handle_t)
252 (vendor_mgt_bind_p->smi_p_bind));
254 /* un register the class */
256 IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl);
257 if (mgt_ret != IB_MGT_OK) {
258 osm_log(p_vend->p_log, OSM_LOG_ERROR,
259 "osm_vendor_destroy: ERR 3C04: "
260 "Fail to unbind the SM class.\n");
263 /* un bind the handle */
264 if (IB_MGT_release_handle
265 (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) {
266 osm_log(p_vend->p_log, OSM_LOG_ERROR,
267 "osm_vendor_destroy: ERR 3C05: "
268 "Fail to unbind the SMI handle.\n");
270 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
271 "osm_vendor_destroy: DBG 1003: "
272 "Unbind the SMI handles.\n");
276 osm_transaction_mgr_destroy(p_vend);
277 /* __osm_mtl_destroy_tid_mad_map( p_vend ); */
278 OSM_LOG_EXIT(p_vend->p_log);
281 /**********************************************************************
282 DEALLOCATE A POINTER TO osm_vendor_t
283 **********************************************************************/
284 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
288 osm_vendor_destroy(*pp_vend);
293 /**********************************************************************
294 * This proc actuall binds the handle to the lower level.
296 * We might have here as a result a casting of our struct to the ib_al_handle_t
298 * Q: Do we need 2 of those - one for MSI and one for GSI ?
299 * A: Yes! We should be able to do the SA too. So we need a struct!
301 **********************************************************************/
304 osm_vendor_init(IN osm_vendor_t * const p_vend,
305 IN osm_log_t * const p_log, IN const uint32_t timeout)
307 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
308 ib_api_status_t status = IB_SUCCESS;
310 OSM_LOG_ENTER(p_log);
312 p_vend->p_log = p_log;
315 * HACK: We need no handle. Assuming the driver is up.
317 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *)
318 malloc(sizeof(osm_vendor_mgt_bind_t));
319 if (ib_mgt_hdl_p == NULL) {
320 osm_log(p_vend->p_log, OSM_LOG_ERROR,
321 "osm_vendor_init: ERR 3C06: "
322 "Fail to allocate vendor mgt handle.\n");
326 ib_mgt_hdl_p->smi_init = FALSE;
327 ib_mgt_hdl_p->gsi_init = FALSE;
328 /* cast it into the ib_al_handle_t h_al */
329 p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p;
330 p_vend->p_transaction_mgr = NULL;
331 osm_transaction_mgr_init(p_vend);
332 /* p_vend->madw_by_tid_map_p = NULL; */
333 /* __osm_mtl_init_tid_mad_map( p_vend ); */
334 p_vend->timeout = timeout;
341 /**********************************************************************
342 * Create and Initialize osm_vendor_t Object
343 **********************************************************************/
344 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
345 IN const uint32_t timeout)
347 ib_api_status_t status;
348 osm_vendor_t *p_vend;
350 OSM_LOG_ENTER(p_log);
354 p_vend = malloc(sizeof(*p_vend));
355 if (p_vend != NULL) {
356 memset(p_vend, 0, sizeof(*p_vend));
357 status = osm_vendor_init(p_vend, p_log, timeout);
358 if (status != IB_SUCCESS) {
359 osm_vendor_delete(&p_vend);
362 osm_log(p_vend->p_log, OSM_LOG_ERROR,
363 "osm_vendor_new: ERR 3C07: "
364 "Fail to allocate vendor object.\n");
371 /**********************************************************************
372 * IB_MGT RCV callback
374 **********************************************************************/
376 __osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,
377 IN void *private_ctx_p,
379 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
382 osm_mtl_bind_info_t *bind_info_p = private_ctx_p;
383 osm_madw_t *req_madw_p = NULL;
385 osm_vend_wrap_t *p_new_vw;
386 osm_mad_addr_t mad_addr;
388 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
390 OSM_LOG_ENTER(p_log);
392 /* if it is a response MAD we mustbe able to get the request */
393 if (ib_mad_is_response((ib_mad_t *) payload_p)) {
394 /* can we find a matching madw by this payload TID */
396 osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend,
397 (ib_mad_t *) payload_p,
399 if (status != IB_MGT_OK) {
400 osm_log(p_log, OSM_LOG_ERROR,
401 "__osm_mtl_rcv_callback: ERR 3C08: "
402 "Error obtaining request madw by TID (%d).\n",
407 if (req_madw_p == NULL) {
408 osm_log(p_log, OSM_LOG_ERROR,
409 "__osm_mtl_rcv_callback: ERR 3C09: "
410 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n",
411 ((ib_mad_t *) payload_p)->method,
412 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id)
419 /* do we have a request ??? */
420 if (req_madw_p == NULL) {
422 /* first arrange an address */
423 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend,
435 osm_log(p_log, OSM_LOG_ERROR,
436 "__osm_mtl_rcv_callback: : "
437 "Received MAD from QP:%X.\n",
438 cl_ntoh32(mad_addr.addr_type.gsi.remote_qp)
441 /* if not - get new osm_madw and arrange it. */
442 /* create the new madw in the pool */
443 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool,
444 (osm_bind_handle_t) bind_info_p,
445 MAD_BLOCK_SIZE, &mad_addr);
446 if (madw_p == NULL) {
447 osm_log(p_log, OSM_LOG_ERROR,
448 "__osm_mtl_rcv_callback: ERR 3C10: "
449 "Error request for a new madw.\n");
452 /* HACK: we cust to avoid the const ??? */
453 mad_buf_p = (void *)madw_p->p_mad;
455 /* we have the madw defined during the send and stored in the vend_wrap */
456 /* we need to make sure the wrapper is correctly init there */
457 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0);
458 madw_p = req_madw_p->vend_wrap.p_resp_madw;
460 /* HACK: we do not Support RMPP */
461 CL_ASSERT(madw_p->h_bind);
463 osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE,
466 if (mad_buf_p == NULL) {
467 osm_log(p_log, OSM_LOG_ERROR,
468 "__osm_mtl_rcv_callback: ERR 3C11: "
469 "Unable to acquire wire MAD.\n");
475 Finally, attach the wire MAD to this wrapper.
477 osm_madw_set_mad(madw_p, mad_buf_p);
479 /* also we need to handle the size of the mad since we did not init ... */
480 madw_p->mad_size = MAD_BLOCK_SIZE;
483 /* init some fields of the vendor wrapper */
484 p_new_vw = osm_madw_get_vend_ptr(madw_p);
485 p_new_vw->h_bind = bind_info_p;
486 p_new_vw->size = MAD_BLOCK_SIZE;
487 p_new_vw->p_resp_madw = NULL;
488 p_new_vw->mad_buf_p = mad_buf_p;
490 /* HACK: We do not support RMPP in receiving MADS */
491 memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE);
493 /* attach the buffer to the wrapper */
494 madw_p->p_mad = mad_buf_p;
496 /* we can also make sure we marked the size and bind on the returned madw */
497 madw_p->h_bind = p_new_vw->h_bind;
500 (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context,
507 /**********************************************************************
508 * IB_MGT Send callback : invoked after each send
510 **********************************************************************/
512 __osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,
514 IN IB_comp_status_t status, IN void *private_ctx_p)
517 osm_mtl_bind_info_t *bind_info_p =
518 (osm_mtl_bind_info_t *) private_ctx_p;
519 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
520 osm_vend_wrap_t *p_vw;
523 OSM_LOG_ENTER(p_log);
525 /* obtain the madp from the wrid */
526 __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p);
528 osm_log(p_log, OSM_LOG_DEBUG,
529 "__osm_mtl_send_callback: INFO 1008: "
530 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
532 /* we need to handle requests and responses differently */
534 if (status != IB_COMP_SUCCESS) {
535 osm_log(p_log, OSM_LOG_ERROR,
536 "__osm_mtl_send_callback: ERR 3C12: "
537 "Error Sending Response MADW:%p.\n", madw_p);
539 osm_log(p_log, OSM_LOG_DEBUG,
540 "__osm_mtl_send_callback: DBG 1008: "
541 "Completed Sending Response MADW:%p.\n",
545 /* if we are a response - we need to clean it up */
546 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
549 /* this call back is invoked on completion of send - error or not */
550 if (status != IB_COMP_SUCCESS) {
552 osm_log(p_log, OSM_LOG_ERROR,
553 "__osm_mtl_send_callback: ERR 3C13: "
554 "Received an Error from IB_MGT Send (%d).\n",
557 p_vw = osm_madw_get_vend_ptr(madw_p);
561 Return any wrappers to the pool that may have been
562 pre-emptively allocated to handle a receive.
564 if (p_vw->p_resp_madw) {
565 osm_mad_pool_put(bind_info_p->p_osm_pool,
567 p_vw->p_resp_madw = NULL;
571 (*bind_info_p->send_err_callback) (bind_info_p->
575 /* successful request send - do nothing - the response will need the
577 osm_log(p_log, OSM_LOG_DEBUG,
578 "__osm_mtl_send_callback: DBG 1008: "
579 "Completed Sending Request MADW:%p.\n", madw_p);
586 /**********************************************************************
587 * BINDs a callback (rcv and send error) for a given class and method
588 * defined by the given: osm_bind_info_t
589 **********************************************************************/
591 osm_vendor_bind(IN osm_vendor_t * const p_vend,
592 IN osm_bind_info_t * const p_user_bind,
593 IN osm_mad_pool_t * const p_mad_pool,
594 IN osm_vend_mad_recv_callback_t mad_recv_callback,
595 IN osm_vend_mad_send_err_callback_t send_err_callback,
598 ib_net64_t port_guid;
599 osm_mtl_bind_info_t *p_bind = NULL;
600 VAPI_hca_hndl_t hca_hndl;
601 VAPI_hca_id_t hca_id;
602 IB_MGT_mad_type_t mad_type;
604 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
605 IB_MGT_ret_t mgt_ret;
607 OSM_LOG_ENTER(p_vend->p_log);
609 CL_ASSERT(p_user_bind);
610 CL_ASSERT(p_mad_pool);
611 CL_ASSERT(mad_recv_callback);
612 CL_ASSERT(send_err_callback);
614 /* cast back the AL handle to vendor mgt bind */
615 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
617 port_guid = p_user_bind->port_guid;
619 osm_log(p_vend->p_log, OSM_LOG_INFO,
621 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
623 /* obtain the hca name and port num from the guid */
624 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
626 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
630 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
632 if (mgt_ret != IB_MGT_OK) {
633 osm_log(p_vend->p_log, OSM_LOG_ERROR,
634 "osm_vendor_bind: ERR 3C14: "
635 "Unable to obtain CA and port (%d).\n");
639 /* create the bind object tracking this binding */
640 p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t));
641 memset(p_bind, 0, sizeof(osm_mtl_bind_info_t));
642 if (p_bind == NULL) {
643 osm_log(p_vend->p_log, OSM_LOG_ERROR,
644 "osm_vendor_bind: ERR 3C15: "
645 "Unable to allocate internal bind object.\n");
649 /* track this bind request info */
650 memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t));
651 p_bind->port_num = port_num;
652 p_bind->p_vend = p_vend;
653 p_bind->client_context = context;
654 p_bind->rcv_callback = mad_recv_callback;
655 p_bind->send_err_callback = send_err_callback;
656 p_bind->p_osm_pool = p_mad_pool;
658 CL_ASSERT(p_bind->port_num);
661 * Get the proper CLASS
664 switch (p_user_bind->mad_class) {
665 case IB_MCLASS_SUBN_LID:
666 case IB_MCLASS_SUBN_DIR:
667 mad_type = IB_MGT_SMI;
670 case IB_MCLASS_SUBN_ADM:
672 mad_type = IB_MGT_GSI;
676 /* we split here - based on the type of MADS GSI / SMI */
677 /* HACK: we only support one class registration per SMI/GSI !!! */
678 if (mad_type == IB_MGT_SMI) {
683 /* we do not need to bind the handle if already available */
684 if (ib_mgt_hdl_p->smi_init == FALSE) {
686 /* First we have to reg and get the handle for the mad */
687 osm_log(p_vend->p_log, OSM_LOG_ERROR,
689 "Binding to IB_MGT SMI of %s port %u\n", hca_id,
693 IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI,
694 &(ib_mgt_hdl_p->smi_mads_hdl));
695 if (IB_MGT_OK != mgt_ret) {
698 osm_log(p_vend->p_log, OSM_LOG_ERROR,
699 "osm_vendor_bind: ERR 3C16: "
700 "Error obtaining IB_MGT handle to SMI.\n");
705 mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl);
706 if (IB_MGT_OK != mgt_ret) {
709 osm_log(p_vend->p_log, OSM_LOG_ERROR,
710 "osm_vendor_bind: ERR 3C17: "
711 "Error binding IB_MGT handle to SM.\n");
715 ib_mgt_hdl_p->smi_init = TRUE;
719 /* attach to this bind info */
720 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl;
721 ib_mgt_hdl_p->smi_p_bind = p_bind;
723 /* now register the callback */
724 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
725 &__osm_mtl_rcv_callback,
727 &__osm_mtl_send_callback,
730 IB_MGT_SEND_CB_MASK);
737 if (ib_mgt_hdl_p->gsi_init == FALSE) {
738 osm_log(p_vend->p_log, OSM_LOG_ERROR,
739 "osm_vendor_bind: " "Binding to IB_MGT GSI\n");
741 /* First we have to reg and get the handle for the mad */
743 IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI,
744 &(ib_mgt_hdl_p->gsi_mads_hdl));
745 if (IB_MGT_OK != mgt_ret) {
748 osm_log(p_vend->p_log, OSM_LOG_ERROR,
749 "osm_vendor_bind: ERR 3C20: "
750 "Error obtaining IB_MGT handle to GSI.\n");
756 IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl,
757 p_user_bind->mad_class);
758 if (IB_MGT_OK != mgt_ret) {
761 osm_log(p_vend->p_log, OSM_LOG_ERROR,
762 "osm_vendor_bind: ERR 3C22: "
763 "Error binding IB_MGT handle to GSI.\n");
767 ib_mgt_hdl_p->gsi_init = TRUE;
769 /* attach to this bind info */
770 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
772 /* now register the callback */
773 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
774 &__osm_mtl_rcv_callback,
776 &__osm_mtl_send_callback,
779 IB_MGT_SEND_CB_MASK);
782 /* we can use the existing handle */
783 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
789 if (IB_MGT_OK != mgt_ret) {
792 osm_log(p_vend->p_log, OSM_LOG_ERROR,
793 "osm_vendor_bind: ERR 3C23: "
794 "Error binding IB_MGT CB (%d).\n", mgt_ret);
798 /* HACK: Do we need to initialize an address vector ???? */
801 OSM_LOG_EXIT(p_vend->p_log);
802 return ((osm_bind_handle_t) p_bind);
805 /**********************************************************************
806 Get a mad from the lower level.
807 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
808 **********************************************************************/
809 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
810 IN const uint32_t mad_size,
811 IN osm_vend_wrap_t * const p_vw)
814 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
815 osm_vendor_t *p_vend = p_bind->p_vend;
817 OSM_LOG_ENTER(p_vend->p_log);
820 /* HACK: We know we can not send through IB_MGT */
821 CL_ASSERT(mad_size <= MAD_BLOCK_SIZE);
823 /* IB_MGT assumes it is 256 - we must follow */
824 p_vw->size = MAD_BLOCK_SIZE;
827 mad_p = (ib_mad_t *) malloc(p_vw->size);
829 osm_log(p_vend->p_log, OSM_LOG_ERROR,
830 "osm_vendor_get: ERR 3C24: "
831 "Error Obtaining MAD buffer.\n");
835 memset(mad_p, 0, p_vw->size);
838 p_vw->mad_buf_p = mad_p;
839 p_vw->h_bind = h_bind;
840 p_vw->p_resp_madw = NULL;
842 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
843 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
845 "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size);
849 OSM_LOG_EXIT(p_vend->p_log);
853 /**********************************************************************
854 * Return a MAD by providing it's wrapper object.
855 **********************************************************************/
857 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
859 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
860 osm_vendor_t *p_vend = p_bind->p_vend;
863 OSM_LOG_ENTER(p_vend->p_log);
866 CL_ASSERT(p_vw->mad_buf_p);
868 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
869 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
870 "osm_vendor_put: " "Retiring MAD %p.\n",
875 * We moved the removal of the transaction to immediatly after
879 /* free the mad but the wrapper is part of the madw object */
880 free(p_vw->mad_buf_p);
881 p_vw->mad_buf_p = NULL;
882 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
883 p_madw->p_mad = NULL;
885 OSM_LOG_EXIT(p_vend->p_log);
888 /**********************************************************************
891 This is for internal use by osm_vendor_send and the transaction mgr
893 **********************************************************************/
895 osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
897 osm_vendor_t *const p_vend = p_bind->p_vend;
898 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
899 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
900 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
901 ib_api_status_t status;
902 IB_MGT_ret_t mgt_res;
907 OSM_LOG_ENTER(p_vend->p_log);
910 * For all sends other than directed route SM MADs,
911 * acquire an address vector for the destination.
913 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
914 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr,
916 IB_MCLASS_SUBN_LID, &av);
918 /* is a directed route - we need to construct a permissive address */
919 memset(&av, 0, sizeof(av));
920 /* we do not need port number since it is part of the mad_hndl */
921 av.dlid = IB_LID_PERMISSIVE;
924 wrid = __osm_set_wrid_by_p_madw(p_madw);
927 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
928 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
931 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
932 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
935 "av.static_rate %d, "
936 "av.path_bits %d.\n",
937 cl_ntoh16(av.dlid), av.static_rate,
941 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad, /* actual payload */
942 &av, /* address vector */
943 wrid, /* casting the mad wrapper pointer for err cb */
947 /* GSI CASE - Support Remote QP */
948 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
949 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
951 "av.dlid 0x%X, av.static_rate %d, "
952 "av.path_bits %d, remote qp: 0x%06X \n",
956 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
960 /* IBMGT have a bug sending to a QP not 1 -
961 the QPN must be in network order except when it qpn 1 ... */
962 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
964 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad, /* actual payload */
965 &av, /* address vector */
966 wrid, /* casting the mad wrapper pointer for err cb */
967 p_vend->timeout, qpn);
970 if (mgt_res != IB_MGT_OK) {
971 osm_log(p_vend->p_log, OSM_LOG_ERROR,
972 "osm_mtl_send_mad: ERR 3C26: "
973 "Error sending mad (%d).\n", mgt_res);
974 if (p_vw->p_resp_madw)
975 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
983 OSM_LOG_EXIT(p_vend->p_log);
987 /**********************************************************************
990 What is unclear to me is the need for the setting of all the MAD Wrapper
991 fields. Seems like the OSM uses these values during it's processing...
992 **********************************************************************/
994 osm_vendor_send(IN osm_bind_handle_t h_bind,
995 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
997 osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind;
998 osm_vendor_t *const p_vend = p_bind->p_vend;
999 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1000 ib_api_status_t status;
1002 OSM_LOG_ENTER(p_vend->p_log);
1005 * If a response is expected to this MAD, then preallocate
1006 * a mad wrapper to contain the wire MAD received in the
1007 * response. Allocating a wrapper here allows for easier
1008 * failure paths than after we already received the wire mad.
1010 if (resp_expected == TRUE) {
1011 /* we track it in the vendor wrapper */
1013 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
1014 if (p_vw->p_resp_madw == NULL) {
1015 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1016 "osm_vendor_send: ERR 3C27: "
1017 "Unable to allocate MAD wrapper.\n");
1018 status = IB_INSUFFICIENT_RESOURCES;
1022 /* put some minimal info on that wrapper */
1023 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
1025 /* we also want to track it in the TID based map */
1026 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t)
1028 if (status != IB_SUCCESS) {
1029 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1030 "osm_vendor_send: ERR 3C25: "
1031 "Error inserting request madw by TID (%d).\n",
1036 p_vw->p_resp_madw = NULL;
1038 /* do the actual send */
1039 status = osm_mtl_send_mad(p_bind, p_madw);
1042 OSM_LOG_EXIT(p_vend->p_log);
1046 /**********************************************************************
1047 * the idea here is to change the content of the bind such that it
1048 * will hold the local address used for sending directed route by the SMA.
1049 **********************************************************************/
1050 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1052 osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend;
1054 OSM_LOG_ENTER(p_vend->p_log);
1056 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
1057 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
1059 OSM_LOG_EXIT(p_vend->p_log);
1061 return (IB_SUCCESS);
1064 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1066 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
1067 osm_vendor_t *p_vend = p_bind->p_vend;
1069 VAPI_hca_attr_t attr_mod;
1070 VAPI_hca_attr_mask_t attr_mask;
1072 OSM_LOG_ENTER(p_vend->p_log);
1074 memset(&attr_mod, 0, sizeof(attr_mod));
1075 memset(&attr_mask, 0, sizeof(attr_mask));
1077 attr_mod.is_sm = is_sm_val;
1078 attr_mask = HCA_ATTR_IS_SM;
1081 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
1083 if (status != VAPI_OK) {
1084 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1085 "osm_vendor_set_sm: ERR 3C28: "
1086 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
1090 OSM_LOG_EXIT(p_vend->p_log);
1093 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1098 #endif /* OSM_VENDOR_INTF_TEST */