2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 #endif /* HAVE_CONFIG_H */
40 #ifdef OSM_VENDOR_INTF_MTL
44 #include <opensm/osm_helper.h>
45 #include <opensm/osm_log.h>
46 /* HACK - I do not know how to prevent complib from loading kernel H files */
48 #include <vendor/osm_vendor_mtl.h>
49 #include <vendor/osm_vendor_api.h>
50 #include <opensm/osm_subnet.h>
51 #include <opensm/osm_opensm.h>
52 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
53 #include <vendor/osm_mtl_bind.h>
56 Since a race can accure on requests. Meaning - a response is received before
57 the send_callback is called - we will save both the madw_p and the fact
58 whether or not it is a response. A race can occure only on requests that did
59 not fail, and then the madw_p will be put back in the pool before the callback.
61 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
65 CL_ASSERT(p_madw->p_mad);
67 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
69 ib_mad_is_response(p_madw->p_mad) |
70 (p_madw->p_mad->method == IB_MAD_METHOD_TRAP_REPRESS);
75 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
76 OUT uint8_t * is_resp,
77 OUT osm_madw_t ** pp_madw)
79 *is_resp = wrid & 0x0000000000000001;
81 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
84 /**********************************************************************
85 * IB_MGT to OSM ADDRESS VECTOR
86 **********************************************************************/
88 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
89 IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
91 OUT osm_mad_addr_t * p_mad_addr)
93 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
94 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
95 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
96 p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
99 p_mad_addr->addr_type.smi.source_lid =
100 cl_hton16(p_rcv_desc->remote_lid);
101 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */
104 /* seems to me there is a IBMGT bug reversing the QPN ... */
105 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
106 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
108 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
109 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
110 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
111 /* the full PKey table - than go by the index. */
112 /* since this does not seem reasonable to me I simply use the default */
113 /* There is a TAVOR limitation that only one P_KEY is supported per */
114 /* QP - so QP1 must use IB_DEFAULT_PKEY */
115 p_mad_addr->addr_type.gsi.pkey_ix = 0;
116 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
118 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
119 /* copy the GRH data if relevant */
120 if (p_mad_addr->addr_type.gsi.global_route) {
121 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
122 ib_grh_set_ver_class_flow(p_rcv_desc->grh.
128 p_mad_addr->addr_type.gsi.grh_info.hop_limit =
129 p_rcv_desc->grh.hop_limit;
130 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
131 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
132 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
133 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
138 /**********************************************************************
139 * OSM ADDR VECTOR TO IB_MGT
140 **********************************************************************/
142 __osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,
143 IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
146 /* For global destination or Multicast address: */
149 memset(p_av, 0, sizeof(IB_ud_av_t));
151 p_av->src_path_bits = p_mad_addr->path_bits;
152 p_av->static_rate = p_mad_addr->static_rate;
153 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
156 p_av->sl = 0; /* Just to note we use 0 here. */
158 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
159 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
161 if (p_mad_addr->addr_type.gsi.global_route) {
162 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
163 grh_info.ver_class_flow, &ver,
164 &p_av->traffic_class,
167 p_mad_addr->addr_type.gsi.grh_info.hop_limit;
168 p_av->sgid_index = 0; /* we always use source GID 0 */
170 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
177 /**********************************************************************
178 **********************************************************************/
179 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
181 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
182 osm_vendor_t *p_vend = p_bind->p_vend;
184 VAPI_hca_attr_t attr_mod;
185 VAPI_hca_attr_mask_t attr_mask;
187 OSM_LOG_ENTER(p_vend->p_log);
189 memset(&attr_mod, 0, sizeof(attr_mod));
190 memset(&attr_mask, 0, sizeof(attr_mask));
192 attr_mod.is_sm = FALSE;
193 attr_mask = HCA_ATTR_IS_SM;
196 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
198 if (status != VAPI_OK) {
199 osm_log(p_vend->p_log, OSM_LOG_ERROR,
200 "__osm_vendor_clear_sm: ERR 3C21: "
201 "Unable set 'IS_SM' bit in port attributes (%d).\n",
205 OSM_LOG_EXIT(p_vend->p_log);
208 /**********************************************************************
209 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
210 **********************************************************************/
211 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
213 memset(p_vend, 0, sizeof(*p_vend));
216 /**********************************************************************
217 * DEALOCATE osm_vendor_t
218 **********************************************************************/
219 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
221 osm_vendor_mgt_bind_t *vendor_mgt_bind_p;
222 IB_MGT_ret_t mgt_ret;
223 OSM_LOG_ENTER(p_vend->p_log);
225 if (p_vend->h_al != NULL) {
226 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
227 if (vendor_mgt_bind_p->gsi_init) {
229 /* un register the class */
230 /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */
232 IB_MGT_unbind_gsi_class(vendor_mgt_bind_p->
235 if (mgt_ret != IB_MGT_OK) {
236 osm_log(p_vend->p_log, OSM_LOG_ERROR,
237 "osm_vendor_destroy: ERR 3C03: "
238 "Fail to unbind the SA class.\n");
241 /* un bind the handle */
242 if (IB_MGT_release_handle
243 (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) {
244 osm_log(p_vend->p_log, OSM_LOG_ERROR,
245 "osm_vendor_destroy: ERR 3C02: "
246 "Fail to unbind the SA GSI handle.\n");
248 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
249 "osm_vendor_destroy: DBG 1002: "
250 "Unbind the GSI handles.\n");
252 if (vendor_mgt_bind_p->smi_init) {
253 /* first - clear the IS_SM in the capability mask */
254 __osm_vendor_clear_sm((osm_bind_handle_t)
255 (vendor_mgt_bind_p->smi_p_bind));
257 /* un register the class */
259 IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl);
260 if (mgt_ret != IB_MGT_OK) {
261 osm_log(p_vend->p_log, OSM_LOG_ERROR,
262 "osm_vendor_destroy: ERR 3C04: "
263 "Fail to unbind the SM class.\n");
266 /* un bind the handle */
267 if (IB_MGT_release_handle
268 (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) {
269 osm_log(p_vend->p_log, OSM_LOG_ERROR,
270 "osm_vendor_destroy: ERR 3C05: "
271 "Fail to unbind the SMI handle.\n");
273 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
274 "osm_vendor_destroy: DBG 1003: "
275 "Unbind the SMI handles.\n");
279 osm_transaction_mgr_destroy(p_vend);
280 /* __osm_mtl_destroy_tid_mad_map( p_vend ); */
281 OSM_LOG_EXIT(p_vend->p_log);
284 /**********************************************************************
285 DEALLOCATE A POINTER TO osm_vendor_t
286 **********************************************************************/
287 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
291 osm_vendor_destroy(*pp_vend);
296 /**********************************************************************
297 * This proc actuall binds the handle to the lower level.
299 * We might have here as a result a casting of our struct to the ib_al_handle_t
301 * Q: Do we need 2 of those - one for MSI and one for GSI ?
302 * A: Yes! We should be able to do the SA too. So we need a struct!
304 **********************************************************************/
307 osm_vendor_init(IN osm_vendor_t * const p_vend,
308 IN osm_log_t * const p_log, IN const uint32_t timeout)
310 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
311 ib_api_status_t status = IB_SUCCESS;
313 OSM_LOG_ENTER(p_log);
315 p_vend->p_log = p_log;
318 * HACK: We need no handle. Assuming the driver is up.
320 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *)
321 malloc(sizeof(osm_vendor_mgt_bind_t));
322 if (ib_mgt_hdl_p == NULL) {
323 osm_log(p_vend->p_log, OSM_LOG_ERROR,
324 "osm_vendor_init: ERR 3C06: "
325 "Fail to allocate vendor mgt handle.\n");
329 ib_mgt_hdl_p->smi_init = FALSE;
330 ib_mgt_hdl_p->gsi_init = FALSE;
331 /* cast it into the ib_al_handle_t h_al */
332 p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p;
333 p_vend->p_transaction_mgr = NULL;
334 osm_transaction_mgr_init(p_vend);
335 /* p_vend->madw_by_tid_map_p = NULL; */
336 /* __osm_mtl_init_tid_mad_map( p_vend ); */
337 p_vend->timeout = timeout;
344 /**********************************************************************
345 * Create and Initialize osm_vendor_t Object
346 **********************************************************************/
347 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
348 IN const uint32_t timeout)
350 ib_api_status_t status;
351 osm_vendor_t *p_vend;
353 OSM_LOG_ENTER(p_log);
357 p_vend = malloc(sizeof(*p_vend));
358 if (p_vend != NULL) {
359 memset(p_vend, 0, sizeof(*p_vend));
360 status = osm_vendor_init(p_vend, p_log, timeout);
361 if (status != IB_SUCCESS) {
362 osm_vendor_delete(&p_vend);
365 osm_log(p_vend->p_log, OSM_LOG_ERROR,
366 "osm_vendor_new: ERR 3C07: "
367 "Fail to allocate vendor object.\n");
374 /**********************************************************************
375 * IB_MGT RCV callback
377 **********************************************************************/
379 __osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,
380 IN void *private_ctx_p,
382 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
385 osm_mtl_bind_info_t *bind_info_p = private_ctx_p;
386 osm_madw_t *req_madw_p = NULL;
388 osm_vend_wrap_t *p_new_vw;
389 osm_mad_addr_t mad_addr;
391 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
393 OSM_LOG_ENTER(p_log);
395 /* if it is a response MAD we mustbe able to get the request */
396 if (ib_mad_is_response((ib_mad_t *) payload_p)) {
397 /* can we find a matching madw by this payload TID */
399 osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend,
400 (ib_mad_t *) payload_p,
402 if (status != IB_MGT_OK) {
403 osm_log(p_log, OSM_LOG_ERROR,
404 "__osm_mtl_rcv_callback: ERR 3C08: "
405 "Error obtaining request madw by TID (%d).\n",
410 if (req_madw_p == NULL) {
411 osm_log(p_log, OSM_LOG_ERROR,
412 "__osm_mtl_rcv_callback: ERR 3C09: "
413 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n",
414 ((ib_mad_t *) payload_p)->method,
415 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id)
422 /* do we have a request ??? */
423 if (req_madw_p == NULL) {
425 /* first arrange an address */
426 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend,
438 osm_log(p_log, OSM_LOG_ERROR,
439 "__osm_mtl_rcv_callback: : "
440 "Received MAD from QP:%X.\n",
441 cl_ntoh32(mad_addr.addr_type.gsi.remote_qp)
444 /* if not - get new osm_madw and arrange it. */
445 /* create the new madw in the pool */
446 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool,
447 (osm_bind_handle_t) bind_info_p,
448 MAD_BLOCK_SIZE, &mad_addr);
449 if (madw_p == NULL) {
450 osm_log(p_log, OSM_LOG_ERROR,
451 "__osm_mtl_rcv_callback: ERR 3C10: "
452 "Error request for a new madw.\n");
455 /* HACK: we cust to avoid the const ??? */
456 mad_buf_p = (void *)madw_p->p_mad;
458 /* we have the madw defined during the send and stored in the vend_wrap */
459 /* we need to make sure the wrapper is correctly init there */
460 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0);
461 madw_p = req_madw_p->vend_wrap.p_resp_madw;
463 /* HACK: we do not Support RMPP */
464 CL_ASSERT(madw_p->h_bind);
466 osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE,
469 if (mad_buf_p == NULL) {
470 osm_log(p_log, OSM_LOG_ERROR,
471 "__osm_mtl_rcv_callback: ERR 3C11: "
472 "Unable to acquire wire MAD.\n");
478 Finally, attach the wire MAD to this wrapper.
480 osm_madw_set_mad(madw_p, mad_buf_p);
482 /* also we need to handle the size of the mad since we did not init ... */
483 madw_p->mad_size = MAD_BLOCK_SIZE;
486 /* init some fields of the vendor wrapper */
487 p_new_vw = osm_madw_get_vend_ptr(madw_p);
488 p_new_vw->h_bind = bind_info_p;
489 p_new_vw->size = MAD_BLOCK_SIZE;
490 p_new_vw->p_resp_madw = NULL;
491 p_new_vw->mad_buf_p = mad_buf_p;
493 /* HACK: We do not support RMPP in receiving MADS */
494 memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE);
496 /* attach the buffer to the wrapper */
497 madw_p->p_mad = mad_buf_p;
499 /* we can also make sure we marked the size and bind on the returned madw */
500 madw_p->h_bind = p_new_vw->h_bind;
503 (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context,
510 /**********************************************************************
511 * IB_MGT Send callback : invoked after each send
513 **********************************************************************/
515 __osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,
517 IN IB_comp_status_t status, IN void *private_ctx_p)
520 osm_mtl_bind_info_t *bind_info_p =
521 (osm_mtl_bind_info_t *) private_ctx_p;
522 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
523 osm_vend_wrap_t *p_vw;
526 OSM_LOG_ENTER(p_log);
528 /* obtain the madp from the wrid */
529 __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p);
531 osm_log(p_log, OSM_LOG_DEBUG,
532 "__osm_mtl_send_callback: INFO 1008: "
533 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
535 /* we need to handle requests and responses differently */
537 if (status != IB_COMP_SUCCESS) {
538 osm_log(p_log, OSM_LOG_ERROR,
539 "__osm_mtl_send_callback: ERR 3C12: "
540 "Error Sending Response MADW:%p.\n", madw_p);
542 osm_log(p_log, OSM_LOG_DEBUG,
543 "__osm_mtl_send_callback: DBG 1008: "
544 "Completed Sending Response MADW:%p.\n",
548 /* if we are a response - we need to clean it up */
549 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
552 /* this call back is invoked on completion of send - error or not */
553 if (status != IB_COMP_SUCCESS) {
555 osm_log(p_log, OSM_LOG_ERROR,
556 "__osm_mtl_send_callback: ERR 3C13: "
557 "Received an Error from IB_MGT Send (%d).\n",
560 p_vw = osm_madw_get_vend_ptr(madw_p);
564 Return any wrappers to the pool that may have been
565 pre-emptively allocated to handle a receive.
567 if (p_vw->p_resp_madw) {
568 osm_mad_pool_put(bind_info_p->p_osm_pool,
570 p_vw->p_resp_madw = NULL;
574 (*bind_info_p->send_err_callback) (bind_info_p->
578 /* successful request send - do nothing - the response will need the
580 osm_log(p_log, OSM_LOG_DEBUG,
581 "__osm_mtl_send_callback: DBG 1008: "
582 "Completed Sending Request MADW:%p.\n", madw_p);
589 /**********************************************************************
590 * BINDs a callback (rcv and send error) for a given class and method
591 * defined by the given: osm_bind_info_t
592 **********************************************************************/
594 osm_vendor_bind(IN osm_vendor_t * const p_vend,
595 IN osm_bind_info_t * const p_user_bind,
596 IN osm_mad_pool_t * const p_mad_pool,
597 IN osm_vend_mad_recv_callback_t mad_recv_callback,
598 IN osm_vend_mad_send_err_callback_t send_err_callback,
601 ib_net64_t port_guid;
602 osm_mtl_bind_info_t *p_bind = NULL;
603 VAPI_hca_hndl_t hca_hndl;
604 VAPI_hca_id_t hca_id;
605 IB_MGT_mad_type_t mad_type;
607 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
608 IB_MGT_ret_t mgt_ret;
610 OSM_LOG_ENTER(p_vend->p_log);
612 CL_ASSERT(p_user_bind);
613 CL_ASSERT(p_mad_pool);
614 CL_ASSERT(mad_recv_callback);
615 CL_ASSERT(send_err_callback);
617 /* cast back the AL handle to vendor mgt bind */
618 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
620 port_guid = p_user_bind->port_guid;
622 osm_log(p_vend->p_log, OSM_LOG_INFO,
624 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
626 /* obtain the hca name and port num from the guid */
627 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
629 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
633 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
635 if (mgt_ret != IB_MGT_OK) {
636 osm_log(p_vend->p_log, OSM_LOG_ERROR,
637 "osm_vendor_bind: ERR 3C14: "
638 "Unable to obtain CA and port (%d).\n");
642 /* create the bind object tracking this binding */
643 p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t));
644 memset(p_bind, 0, sizeof(osm_mtl_bind_info_t));
645 if (p_bind == NULL) {
646 osm_log(p_vend->p_log, OSM_LOG_ERROR,
647 "osm_vendor_bind: ERR 3C15: "
648 "Unable to allocate internal bind object.\n");
652 /* track this bind request info */
653 memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t));
654 p_bind->port_num = port_num;
655 p_bind->p_vend = p_vend;
656 p_bind->client_context = context;
657 p_bind->rcv_callback = mad_recv_callback;
658 p_bind->send_err_callback = send_err_callback;
659 p_bind->p_osm_pool = p_mad_pool;
661 CL_ASSERT(p_bind->port_num);
664 * Get the proper CLASS
667 switch (p_user_bind->mad_class) {
668 case IB_MCLASS_SUBN_LID:
669 case IB_MCLASS_SUBN_DIR:
670 mad_type = IB_MGT_SMI;
673 case IB_MCLASS_SUBN_ADM:
675 mad_type = IB_MGT_GSI;
679 /* we split here - based on the type of MADS GSI / SMI */
680 /* HACK: we only support one class registration per SMI/GSI !!! */
681 if (mad_type == IB_MGT_SMI) {
686 /* we do not need to bind the handle if already available */
687 if (ib_mgt_hdl_p->smi_init == FALSE) {
689 /* First we have to reg and get the handle for the mad */
690 osm_log(p_vend->p_log, OSM_LOG_ERROR,
692 "Binding to IB_MGT SMI of %s port %u\n", hca_id,
696 IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI,
697 &(ib_mgt_hdl_p->smi_mads_hdl));
698 if (IB_MGT_OK != mgt_ret) {
701 osm_log(p_vend->p_log, OSM_LOG_ERROR,
702 "osm_vendor_bind: ERR 3C16: "
703 "Error obtaining IB_MGT handle to SMI.\n");
708 mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl);
709 if (IB_MGT_OK != mgt_ret) {
712 osm_log(p_vend->p_log, OSM_LOG_ERROR,
713 "osm_vendor_bind: ERR 3C17: "
714 "Error binding IB_MGT handle to SM.\n");
718 ib_mgt_hdl_p->smi_init = TRUE;
722 /* attach to this bind info */
723 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl;
724 ib_mgt_hdl_p->smi_p_bind = p_bind;
726 /* now register the callback */
727 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
728 &__osm_mtl_rcv_callback,
730 &__osm_mtl_send_callback,
733 IB_MGT_SEND_CB_MASK);
740 if (ib_mgt_hdl_p->gsi_init == FALSE) {
741 osm_log(p_vend->p_log, OSM_LOG_ERROR,
742 "osm_vendor_bind: " "Binding to IB_MGT GSI\n");
744 /* First we have to reg and get the handle for the mad */
746 IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI,
747 &(ib_mgt_hdl_p->gsi_mads_hdl));
748 if (IB_MGT_OK != mgt_ret) {
751 osm_log(p_vend->p_log, OSM_LOG_ERROR,
752 "osm_vendor_bind: ERR 3C20: "
753 "Error obtaining IB_MGT handle to GSI.\n");
759 IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl,
760 p_user_bind->mad_class);
761 if (IB_MGT_OK != mgt_ret) {
764 osm_log(p_vend->p_log, OSM_LOG_ERROR,
765 "osm_vendor_bind: ERR 3C22: "
766 "Error binding IB_MGT handle to GSI.\n");
770 ib_mgt_hdl_p->gsi_init = TRUE;
772 /* attach to this bind info */
773 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
775 /* now register the callback */
776 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
777 &__osm_mtl_rcv_callback,
779 &__osm_mtl_send_callback,
782 IB_MGT_SEND_CB_MASK);
785 /* we can use the existing handle */
786 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
792 if (IB_MGT_OK != mgt_ret) {
795 osm_log(p_vend->p_log, OSM_LOG_ERROR,
796 "osm_vendor_bind: ERR 3C23: "
797 "Error binding IB_MGT CB (%d).\n", mgt_ret);
801 /* HACK: Do we need to initialize an address vector ???? */
804 OSM_LOG_EXIT(p_vend->p_log);
805 return ((osm_bind_handle_t) p_bind);
808 /**********************************************************************
809 Get a mad from the lower level.
810 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
811 **********************************************************************/
812 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
813 IN const uint32_t mad_size,
814 IN osm_vend_wrap_t * const p_vw)
817 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
818 osm_vendor_t *p_vend = p_bind->p_vend;
820 OSM_LOG_ENTER(p_vend->p_log);
823 /* HACK: We know we can not send through IB_MGT */
824 CL_ASSERT(mad_size <= MAD_BLOCK_SIZE);
826 /* IB_MGT assumes it is 256 - we must follow */
827 p_vw->size = MAD_BLOCK_SIZE;
830 mad_p = (ib_mad_t *) malloc(p_vw->size);
832 osm_log(p_vend->p_log, OSM_LOG_ERROR,
833 "osm_vendor_get: ERR 3C24: "
834 "Error Obtaining MAD buffer.\n");
838 memset(mad_p, 0, p_vw->size);
841 p_vw->mad_buf_p = mad_p;
842 p_vw->h_bind = h_bind;
843 p_vw->p_resp_madw = NULL;
845 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
846 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
848 "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size);
852 OSM_LOG_EXIT(p_vend->p_log);
856 /**********************************************************************
857 * Return a MAD by providing it's wrapper object.
858 **********************************************************************/
860 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
862 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
863 osm_vendor_t *p_vend = p_bind->p_vend;
866 OSM_LOG_ENTER(p_vend->p_log);
869 CL_ASSERT(p_vw->mad_buf_p);
871 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
872 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
873 "osm_vendor_put: " "Retiring MAD %p.\n",
878 * We moved the removal of the transaction to immediatly after
882 /* free the mad but the wrapper is part of the madw object */
883 free(p_vw->mad_buf_p);
884 p_vw->mad_buf_p = NULL;
885 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
886 p_madw->p_mad = NULL;
888 OSM_LOG_EXIT(p_vend->p_log);
891 /**********************************************************************
894 This is for internal use by osm_vendor_send and the transaction mgr
896 **********************************************************************/
898 osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
900 osm_vendor_t *const p_vend = p_bind->p_vend;
901 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
902 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
903 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
904 ib_api_status_t status;
905 IB_MGT_ret_t mgt_res;
910 OSM_LOG_ENTER(p_vend->p_log);
913 * For all sends other than directed route SM MADs,
914 * acquire an address vector for the destination.
916 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
917 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr,
919 IB_MCLASS_SUBN_LID, &av);
921 /* is a directed route - we need to construct a permissive address */
922 memset(&av, 0, sizeof(av));
923 /* we do not need port number since it is part of the mad_hndl */
924 av.dlid = IB_LID_PERMISSIVE;
927 wrid = __osm_set_wrid_by_p_madw(p_madw);
930 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
931 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
934 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
935 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
938 "av.static_rate %d, "
939 "av.path_bits %d.\n",
940 cl_ntoh16(av.dlid), av.static_rate,
944 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad, /* actual payload */
945 &av, /* address vector */
946 wrid, /* casting the mad wrapper pointer for err cb */
950 /* GSI CASE - Support Remote QP */
951 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
952 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
954 "av.dlid 0x%X, av.static_rate %d, "
955 "av.path_bits %d, remote qp: 0x%06X \n",
959 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
963 /* IBMGT have a bug sending to a QP not 1 -
964 the QPN must be in network order except when it qpn 1 ... */
965 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
967 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad, /* actual payload */
968 &av, /* address vector */
969 wrid, /* casting the mad wrapper pointer for err cb */
970 p_vend->timeout, qpn);
973 if (mgt_res != IB_MGT_OK) {
974 osm_log(p_vend->p_log, OSM_LOG_ERROR,
975 "osm_mtl_send_mad: ERR 3C26: "
976 "Error sending mad (%d).\n", mgt_res);
977 if (p_vw->p_resp_madw)
978 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
986 OSM_LOG_EXIT(p_vend->p_log);
990 /**********************************************************************
993 What is unclear to me is the need for the setting of all the MAD Wrapper
994 fields. Seems like the OSM uses these values during it's processing...
995 **********************************************************************/
997 osm_vendor_send(IN osm_bind_handle_t h_bind,
998 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
1000 osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind;
1001 osm_vendor_t *const p_vend = p_bind->p_vend;
1002 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1003 ib_api_status_t status;
1005 OSM_LOG_ENTER(p_vend->p_log);
1008 * If a response is expected to this MAD, then preallocate
1009 * a mad wrapper to contain the wire MAD received in the
1010 * response. Allocating a wrapper here allows for easier
1011 * failure paths than after we already received the wire mad.
1013 if (resp_expected == TRUE) {
1014 /* we track it in the vendor wrapper */
1016 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
1017 if (p_vw->p_resp_madw == NULL) {
1018 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1019 "osm_vendor_send: ERR 3C27: "
1020 "Unable to allocate MAD wrapper.\n");
1021 status = IB_INSUFFICIENT_RESOURCES;
1025 /* put some minimal info on that wrapper */
1026 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
1028 /* we also want to track it in the TID based map */
1029 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t)
1031 if (status != IB_SUCCESS) {
1032 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1033 "osm_vendor_send: ERR 3C25: "
1034 "Error inserting request madw by TID (%d).\n",
1039 p_vw->p_resp_madw = NULL;
1041 /* do the actual send */
1042 status = osm_mtl_send_mad(p_bind, p_madw);
1045 OSM_LOG_EXIT(p_vend->p_log);
1049 /**********************************************************************
1050 * the idea here is to change the content of the bind such that it
1051 * will hold the local address used for sending directed route by the SMA.
1052 **********************************************************************/
1053 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1055 osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend;
1057 OSM_LOG_ENTER(p_vend->p_log);
1059 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
1060 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
1062 OSM_LOG_EXIT(p_vend->p_log);
1064 return (IB_SUCCESS);
1067 /**********************************************************************
1068 **********************************************************************/
1069 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1071 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
1072 osm_vendor_t *p_vend = p_bind->p_vend;
1074 VAPI_hca_attr_t attr_mod;
1075 VAPI_hca_attr_mask_t attr_mask;
1077 OSM_LOG_ENTER(p_vend->p_log);
1079 memset(&attr_mod, 0, sizeof(attr_mod));
1080 memset(&attr_mask, 0, sizeof(attr_mask));
1082 attr_mod.is_sm = is_sm_val;
1083 attr_mask = HCA_ATTR_IS_SM;
1086 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
1088 if (status != VAPI_OK) {
1089 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1090 "osm_vendor_set_sm: ERR 3C28: "
1091 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
1095 OSM_LOG_EXIT(p_vend->p_log);
1098 /**********************************************************************
1099 **********************************************************************/
1100 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1105 #endif /* OSM_VENDOR_INTF_TEST */