]> CyberLeo.Net >> Repos - FreeBSD/releng/9.2.git/blob - contrib/ofed/management/opensm/libvendor/osm_vendor_mtl.c
- Copy stable/9 to releng/9.2 as part of the 9.2-RELEASE cycle.
[FreeBSD/releng/9.2.git] / contrib / ofed / management / opensm / libvendor / osm_vendor_mtl.c
1 /*
2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #if HAVE_CONFIG_H
37 #  include <config.h>
38 #endif                          /* HAVE_CONFIG_H */
39
40 #ifdef OSM_VENDOR_INTF_MTL
41
42 #include <stdlib.h>
43 #include <string.h>
44 #include <opensm/osm_helper.h>
45 #include <opensm/osm_log.h>
46 /* HACK - I do not know how to prevent complib from loading kernel H files */
47 #undef __init
48 #include <vendor/osm_vendor_mtl.h>
49 #include <vendor/osm_vendor_api.h>
50 #include <opensm/osm_subnet.h>
51 #include <opensm/osm_opensm.h>
52 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
53 #include <vendor/osm_mtl_bind.h>
54
55 /*
56   Since a race can accure on requests. Meaning - a response is received before
57   the send_callback is called - we will save both the madw_p and the fact
58   whether or not it is a response. A race can occure only on requests that did
59   not fail, and then the madw_p will be put back in the pool before the callback.
60 */
61 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
62 {
63         uint64_t wrid = 0;
64
65         CL_ASSERT(p_madw->p_mad);
66
67         memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
68         wrid = (wrid << 1) |
69             ib_mad_is_response(p_madw->p_mad) |
70             (p_madw->p_mad->method == IB_MAD_METHOD_TRAP_REPRESS);
71         return wrid;
72 }
73
74 void
75 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
76                                   OUT uint8_t * is_resp,
77                                   OUT osm_madw_t ** pp_madw)
78 {
79         *is_resp = wrid & 0x0000000000000001;
80         wrid = wrid >> 1;
81         memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
82 }
83
84 /**********************************************************************
85  * IB_MGT to OSM ADDRESS VECTOR
86  **********************************************************************/
87 void
88 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
89                                           IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
90                                           IN uint8_t is_smi,
91                                           OUT osm_mad_addr_t * p_mad_addr)
92 {
93         /*  p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
94         p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
95         p_mad_addr->static_rate = 0;    /*  HACK - we do not  know the rate ! */
96         p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
97         if (is_smi) {
98                 /* SMI */
99                 p_mad_addr->addr_type.smi.source_lid =
100                     cl_hton16(p_rcv_desc->remote_lid);
101                 p_mad_addr->addr_type.smi.port_num = 99;        /*  HACK - if used - should fail */
102         } else {
103                 /* GSI */
104                 /* seems to me there is a IBMGT bug reversing the QPN ... */
105                 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
106                 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
107
108                 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
109                 /*  we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
110                 /*  the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
111                 /*  the full PKey table - than go by the index. */
112                 /*  since this does not seem reasonable to me I simply use the default */
113                 /*  There is a TAVOR limitation that only one P_KEY is supported per  */
114                 /*  QP - so QP1 must use IB_DEFAULT_PKEY */
115                 p_mad_addr->addr_type.gsi.pkey_ix = 0;
116                 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
117
118                 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
119                 /* copy the GRH data if relevant */
120                 if (p_mad_addr->addr_type.gsi.global_route) {
121                         p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
122                             ib_grh_set_ver_class_flow(p_rcv_desc->grh.
123                                                       IP_version,
124                                                       p_rcv_desc->grh.
125                                                       traffic_class,
126                                                       p_rcv_desc->grh.
127                                                       flow_label);
128                         p_mad_addr->addr_type.gsi.grh_info.hop_limit =
129                             p_rcv_desc->grh.hop_limit;
130                         memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
131                                &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
132                         memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
133                                p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
134                 }
135         }
136 }
137
138 /**********************************************************************
139  * OSM ADDR VECTOR TO IB_MGT
140  **********************************************************************/
141 void
142 __osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,
143                                       IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
144 {
145
146         /* For global destination or Multicast address: */
147         u_int8_t ver;
148
149         memset(p_av, 0, sizeof(IB_ud_av_t));
150
151         p_av->src_path_bits = p_mad_addr->path_bits;
152         p_av->static_rate = p_mad_addr->static_rate;
153         p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
154
155         if (is_smi) {
156                 p_av->sl = 0;   /*  Just to note we use 0 here. */
157         } else {
158                 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
159                 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
160
161                 if (p_mad_addr->addr_type.gsi.global_route) {
162                         ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
163                                                   grh_info.ver_class_flow, &ver,
164                                                   &p_av->traffic_class,
165                                                   &p_av->flow_label);
166                         p_av->hop_limit =
167                             p_mad_addr->addr_type.gsi.grh_info.hop_limit;
168                         p_av->sgid_index = 0;   /*  we always use source GID 0 */
169                         memcpy(&p_av->dgid,
170                                &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
171                                sizeof(ib_net64_t));
172
173                 }
174         }
175 }
176
177 /**********************************************************************
178  **********************************************************************/
179 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
180 {
181         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
182         osm_vendor_t *p_vend = p_bind->p_vend;
183         VAPI_ret_t status;
184         VAPI_hca_attr_t attr_mod;
185         VAPI_hca_attr_mask_t attr_mask;
186
187         OSM_LOG_ENTER(p_vend->p_log);
188
189         memset(&attr_mod, 0, sizeof(attr_mod));
190         memset(&attr_mask, 0, sizeof(attr_mask));
191
192         attr_mod.is_sm = FALSE;
193         attr_mask = HCA_ATTR_IS_SM;
194
195         status =
196             VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
197                                  &attr_mask);
198         if (status != VAPI_OK) {
199                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
200                         "__osm_vendor_clear_sm: ERR 3C21: "
201                         "Unable set 'IS_SM' bit in port attributes (%d).\n",
202                         status);
203         }
204
205         OSM_LOG_EXIT(p_vend->p_log);
206 }
207
208 /**********************************************************************
209  * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
210  **********************************************************************/
211 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
212 {
213         memset(p_vend, 0, sizeof(*p_vend));
214 }
215
216 /**********************************************************************
217  * DEALOCATE osm_vendor_t
218  **********************************************************************/
219 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
220 {
221         osm_vendor_mgt_bind_t *vendor_mgt_bind_p;
222         IB_MGT_ret_t mgt_ret;
223         OSM_LOG_ENTER(p_vend->p_log);
224
225         if (p_vend->h_al != NULL) {
226                 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
227                 if (vendor_mgt_bind_p->gsi_init) {
228
229                         /* un register the class */
230                         /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */
231                         mgt_ret =
232                             IB_MGT_unbind_gsi_class(vendor_mgt_bind_p->
233                                                     gsi_mads_hdl,
234                                                     IB_MCLASS_SUBN_ADM);
235                         if (mgt_ret != IB_MGT_OK) {
236                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
237                                         "osm_vendor_destroy: ERR 3C03: "
238                                         "Fail to unbind the SA class.\n");
239                         }
240
241                         /* un bind the handle */
242                         if (IB_MGT_release_handle
243                             (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) {
244                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
245                                         "osm_vendor_destroy: ERR 3C02: "
246                                         "Fail to unbind the SA GSI handle.\n");
247                         }
248                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
249                                 "osm_vendor_destroy: DBG 1002: "
250                                 "Unbind the GSI handles.\n");
251                 }
252                 if (vendor_mgt_bind_p->smi_init) {
253                         /* first - clear the IS_SM in the capability mask */
254                         __osm_vendor_clear_sm((osm_bind_handle_t)
255                                               (vendor_mgt_bind_p->smi_p_bind));
256
257                         /* un register the class */
258                         mgt_ret =
259                             IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl);
260                         if (mgt_ret != IB_MGT_OK) {
261                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
262                                         "osm_vendor_destroy: ERR 3C04: "
263                                         "Fail to unbind the SM class.\n");
264                         }
265
266                         /* un bind the handle */
267                         if (IB_MGT_release_handle
268                             (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) {
269                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
270                                         "osm_vendor_destroy: ERR 3C05: "
271                                         "Fail to unbind the SMI handle.\n");
272                         }
273                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
274                                 "osm_vendor_destroy: DBG 1003: "
275                                 "Unbind the SMI handles.\n");
276
277                 }
278         }
279         osm_transaction_mgr_destroy(p_vend);
280         /*  __osm_mtl_destroy_tid_mad_map( p_vend ); */
281         OSM_LOG_EXIT(p_vend->p_log);
282 }
283
284 /**********************************************************************
285 DEALLOCATE A POINTER TO osm_vendor_t
286 **********************************************************************/
287 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
288 {
289         CL_ASSERT(pp_vend);
290
291         osm_vendor_destroy(*pp_vend);
292         free(*pp_vend);
293         *pp_vend = NULL;
294 }
295
296 /**********************************************************************
297  * This proc actuall binds the handle to the lower level.
298  *
299  * We might have here as a result a casting of our struct to the ib_al_handle_t
300  *
301  * Q: Do we need 2 of those - one for MSI and one for GSI ?
302  * A: Yes! We should be able to do the SA too. So we need a struct!
303  *
304  **********************************************************************/
305
306 ib_api_status_t
307 osm_vendor_init(IN osm_vendor_t * const p_vend,
308                 IN osm_log_t * const p_log, IN const uint32_t timeout)
309 {
310         osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
311         ib_api_status_t status = IB_SUCCESS;
312
313         OSM_LOG_ENTER(p_log);
314
315         p_vend->p_log = p_log;
316
317         /*
318          * HACK: We need no handle. Assuming the driver is up.
319          */
320         ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *)
321             malloc(sizeof(osm_vendor_mgt_bind_t));
322         if (ib_mgt_hdl_p == NULL) {
323                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
324                         "osm_vendor_init: ERR 3C06: "
325                         "Fail to allocate vendor mgt handle.\n");
326                 goto Exit;
327         }
328
329         ib_mgt_hdl_p->smi_init = FALSE;
330         ib_mgt_hdl_p->gsi_init = FALSE;
331         /* cast it into the ib_al_handle_t h_al */
332         p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p;
333         p_vend->p_transaction_mgr = NULL;
334         osm_transaction_mgr_init(p_vend);
335         /*  p_vend->madw_by_tid_map_p = NULL; */
336         /*  __osm_mtl_init_tid_mad_map( p_vend ); */
337         p_vend->timeout = timeout;
338
339 Exit:
340         OSM_LOG_EXIT(p_log);
341         return (status);
342 }
343
344 /**********************************************************************
345  *  Create and Initialize osm_vendor_t Object
346  **********************************************************************/
347 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
348                              IN const uint32_t timeout)
349 {
350         ib_api_status_t status;
351         osm_vendor_t *p_vend;
352
353         OSM_LOG_ENTER(p_log);
354
355         CL_ASSERT(p_log);
356
357         p_vend = malloc(sizeof(*p_vend));
358         if (p_vend != NULL) {
359                 memset(p_vend, 0, sizeof(*p_vend));
360                 status = osm_vendor_init(p_vend, p_log, timeout);
361                 if (status != IB_SUCCESS) {
362                         osm_vendor_delete(&p_vend);
363                 }
364         } else {
365                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
366                         "osm_vendor_new: ERR 3C07: "
367                         "Fail to allocate vendor object.\n");
368         }
369
370         OSM_LOG_EXIT(p_log);
371         return (p_vend);
372 }
373
374 /**********************************************************************
375  * IB_MGT RCV callback
376  *
377  **********************************************************************/
378 void
379 __osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,
380                        IN void *private_ctx_p,
381                        IN void *payload_p,
382                        IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
383 {
384         IB_MGT_ret_t status;
385         osm_mtl_bind_info_t *bind_info_p = private_ctx_p;
386         osm_madw_t *req_madw_p = NULL;
387         osm_madw_t *madw_p;
388         osm_vend_wrap_t *p_new_vw;
389         osm_mad_addr_t mad_addr;
390         ib_mad_t *mad_buf_p;
391         osm_log_t *const p_log = bind_info_p->p_vend->p_log;
392
393         OSM_LOG_ENTER(p_log);
394
395         /* if it is a response MAD we mustbe able to get the request */
396         if (ib_mad_is_response((ib_mad_t *) payload_p)) {
397                 /* can we find a matching madw by this payload TID */
398                 status =
399                     osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend,
400                                                          (ib_mad_t *) payload_p,
401                                                          &req_madw_p);
402                 if (status != IB_MGT_OK) {
403                         osm_log(p_log, OSM_LOG_ERROR,
404                                 "__osm_mtl_rcv_callback: ERR 3C08: "
405                                 "Error obtaining request madw by TID (%d).\n",
406                                 status);
407                         req_madw_p = NULL;
408                 }
409
410                 if (req_madw_p == NULL) {
411                         osm_log(p_log, OSM_LOG_ERROR,
412                                 "__osm_mtl_rcv_callback: ERR 3C09:  "
413                                 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n",
414                                 ((ib_mad_t *) payload_p)->method,
415                                 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id)
416
417                             );
418                         goto Exit;
419                 }
420         }
421
422         /* do we have a request ??? */
423         if (req_madw_p == NULL) {
424
425                 /* first arrange an address */
426                 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend,
427                                                           rcv_remote_info_p,
428                                                           (((ib_mad_t *)
429                                                             payload_p)->
430                                                            mgmt_class ==
431                                                            IB_MCLASS_SUBN_LID)
432                                                           || (((ib_mad_t *)
433                                                                payload_p)->
434                                                               mgmt_class ==
435                                                               IB_MCLASS_SUBN_DIR),
436                                                           &mad_addr);
437
438                 osm_log(p_log, OSM_LOG_ERROR,
439                         "__osm_mtl_rcv_callback: : "
440                         "Received MAD from QP:%X.\n",
441                         cl_ntoh32(mad_addr.addr_type.gsi.remote_qp)
442                     );
443
444                 /* if not - get new osm_madw and arrange it. */
445                 /* create the new madw in the pool */
446                 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool,
447                                           (osm_bind_handle_t) bind_info_p,
448                                           MAD_BLOCK_SIZE, &mad_addr);
449                 if (madw_p == NULL) {
450                         osm_log(p_log, OSM_LOG_ERROR,
451                                 "__osm_mtl_rcv_callback: ERR 3C10: "
452                                 "Error request for a new madw.\n");
453                         goto Exit;
454                 }
455                 /* HACK: we cust to avoid the const ??? */
456                 mad_buf_p = (void *)madw_p->p_mad;
457         } else {
458                 /* we have the madw defined during the send and stored in the vend_wrap */
459                 /* we need to make sure the wrapper is correctly init there */
460                 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0);
461                 madw_p = req_madw_p->vend_wrap.p_resp_madw;
462
463                 /* HACK: we do not Support RMPP */
464                 CL_ASSERT(madw_p->h_bind);
465                 mad_buf_p =
466                     osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE,
467                                    &madw_p->vend_wrap);
468
469                 if (mad_buf_p == NULL) {
470                         osm_log(p_log, OSM_LOG_ERROR,
471                                 "__osm_mtl_rcv_callback: ERR 3C11: "
472                                 "Unable to acquire wire MAD.\n");
473
474                         goto Exit;
475                 }
476
477                 /*
478                    Finally, attach the wire MAD to this wrapper.
479                  */
480                 osm_madw_set_mad(madw_p, mad_buf_p);
481
482                 /* also we need to handle the size of the mad since we did not init ... */
483                 madw_p->mad_size = MAD_BLOCK_SIZE;
484         }
485
486         /* init some fields of the vendor wrapper */
487         p_new_vw = osm_madw_get_vend_ptr(madw_p);
488         p_new_vw->h_bind = bind_info_p;
489         p_new_vw->size = MAD_BLOCK_SIZE;
490         p_new_vw->p_resp_madw = NULL;
491         p_new_vw->mad_buf_p = mad_buf_p;
492
493         /* HACK: We do not support RMPP in receiving MADS */
494         memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE);
495
496         /* attach the buffer to the wrapper */
497         madw_p->p_mad = mad_buf_p;
498
499         /* we can also make sure we marked the size and bind on the returned madw */
500         madw_p->h_bind = p_new_vw->h_bind;
501
502         /* call the CB */
503         (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context,
504                                       req_madw_p);
505
506 Exit:
507         OSM_LOG_EXIT(p_log);
508 }
509
510 /**********************************************************************
511  * IB_MGT Send callback : invoked after each send
512  *
513  **********************************************************************/
514 void
515 __osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,
516                         IN u_int64_t wrid,
517                         IN IB_comp_status_t status, IN void *private_ctx_p)
518 {
519         osm_madw_t *madw_p;
520         osm_mtl_bind_info_t *bind_info_p =
521             (osm_mtl_bind_info_t *) private_ctx_p;
522         osm_log_t *const p_log = bind_info_p->p_vend->p_log;
523         osm_vend_wrap_t *p_vw;
524         uint8_t is_resp;
525
526         OSM_LOG_ENTER(p_log);
527
528         /* obtain the madp from the wrid */
529         __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p);
530
531         osm_log(p_log, OSM_LOG_DEBUG,
532                 "__osm_mtl_send_callback: INFO 1008: "
533                 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
534
535         /* we need to handle requests and responses differently */
536         if (is_resp) {
537                 if (status != IB_COMP_SUCCESS) {
538                         osm_log(p_log, OSM_LOG_ERROR,
539                                 "__osm_mtl_send_callback: ERR 3C12: "
540                                 "Error Sending Response MADW:%p.\n", madw_p);
541                 } else {
542                         osm_log(p_log, OSM_LOG_DEBUG,
543                                 "__osm_mtl_send_callback: DBG 1008: "
544                                 "Completed Sending Response MADW:%p.\n",
545                                 madw_p);
546                 }
547
548                 /* if we are a response - we need to clean it up */
549                 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
550         } else {
551
552                 /* this call back is invoked on completion of send - error or not */
553                 if (status != IB_COMP_SUCCESS) {
554
555                         osm_log(p_log, OSM_LOG_ERROR,
556                                 "__osm_mtl_send_callback: ERR 3C13: "
557                                 "Received an Error from IB_MGT Send (%d).\n",
558                                 status);
559
560                         p_vw = osm_madw_get_vend_ptr(madw_p);
561                         CL_ASSERT(p_vw);
562
563                         /*
564                            Return any wrappers to the pool that may have been
565                            pre-emptively allocated to handle a receive.
566                          */
567                         if (p_vw->p_resp_madw) {
568                                 osm_mad_pool_put(bind_info_p->p_osm_pool,
569                                                  p_vw->p_resp_madw);
570                                 p_vw->p_resp_madw = NULL;
571                         }
572
573                         /* invoke the CB */
574                         (*bind_info_p->send_err_callback) (bind_info_p->
575                                                            client_context,
576                                                            madw_p);
577                 } else {
578                         /* successful request send - do nothing - the response will need the
579                            out mad */
580                         osm_log(p_log, OSM_LOG_DEBUG,
581                                 "__osm_mtl_send_callback: DBG 1008: "
582                                 "Completed Sending Request MADW:%p.\n", madw_p);
583                 }
584         }
585
586         OSM_LOG_EXIT(p_log);
587 }
588
589 /**********************************************************************
590  * BINDs a callback (rcv and send error) for a given class and method
591  * defined by the given:  osm_bind_info_t
592  **********************************************************************/
593 osm_bind_handle_t
594 osm_vendor_bind(IN osm_vendor_t * const p_vend,
595                 IN osm_bind_info_t * const p_user_bind,
596                 IN osm_mad_pool_t * const p_mad_pool,
597                 IN osm_vend_mad_recv_callback_t mad_recv_callback,
598                 IN osm_vend_mad_send_err_callback_t send_err_callback,
599                 IN void *context)
600 {
601         ib_net64_t port_guid;
602         osm_mtl_bind_info_t *p_bind = NULL;
603         VAPI_hca_hndl_t hca_hndl;
604         VAPI_hca_id_t hca_id;
605         IB_MGT_mad_type_t mad_type;
606         uint32_t port_num;
607         osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
608         IB_MGT_ret_t mgt_ret;
609
610         OSM_LOG_ENTER(p_vend->p_log);
611
612         CL_ASSERT(p_user_bind);
613         CL_ASSERT(p_mad_pool);
614         CL_ASSERT(mad_recv_callback);
615         CL_ASSERT(send_err_callback);
616
617         /* cast back the AL handle to vendor mgt bind */
618         ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
619
620         port_guid = p_user_bind->port_guid;
621
622         osm_log(p_vend->p_log, OSM_LOG_INFO,
623                 "osm_vendor_bind: "
624                 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
625
626         /* obtain the hca name and port num from the guid */
627         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
628                 "osm_vendor_bind: "
629                 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
630                 port_guid);
631
632         mgt_ret =
633             osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
634                                             &hca_id, &port_num);
635         if (mgt_ret != IB_MGT_OK) {
636                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
637                         "osm_vendor_bind: ERR 3C14: "
638                         "Unable to obtain CA and port (%d).\n");
639                 goto Exit;
640         }
641
642         /* create the bind object tracking this binding */
643         p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t));
644         memset(p_bind, 0, sizeof(osm_mtl_bind_info_t));
645         if (p_bind == NULL) {
646                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
647                         "osm_vendor_bind: ERR 3C15: "
648                         "Unable to allocate internal bind object.\n");
649                 goto Exit;
650         }
651
652         /* track this bind request info */
653         memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t));
654         p_bind->port_num = port_num;
655         p_bind->p_vend = p_vend;
656         p_bind->client_context = context;
657         p_bind->rcv_callback = mad_recv_callback;
658         p_bind->send_err_callback = send_err_callback;
659         p_bind->p_osm_pool = p_mad_pool;
660
661         CL_ASSERT(p_bind->port_num);
662
663         /*
664          * Get the proper CLASS
665          */
666
667         switch (p_user_bind->mad_class) {
668         case IB_MCLASS_SUBN_LID:
669         case IB_MCLASS_SUBN_DIR:
670                 mad_type = IB_MGT_SMI;
671                 break;
672
673         case IB_MCLASS_SUBN_ADM:
674         default:
675                 mad_type = IB_MGT_GSI;
676                 break;
677         }
678
679         /* we split here - based on the type of MADS GSI / SMI */
680         /* HACK: we only support one class registration per SMI/GSI !!! */
681         if (mad_type == IB_MGT_SMI) {
682                 /*
683                  *  SMI CASE
684                  */
685
686                 /* we do not need to bind the handle if already available */
687                 if (ib_mgt_hdl_p->smi_init == FALSE) {
688
689                         /* First we have to reg and get the handle for the mad */
690                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
691                                 "osm_vendor_bind: "
692                                 "Binding to IB_MGT SMI of %s port %u\n", hca_id,
693                                 port_num);
694
695                         mgt_ret =
696                             IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI,
697                                               &(ib_mgt_hdl_p->smi_mads_hdl));
698                         if (IB_MGT_OK != mgt_ret) {
699                                 free(p_bind);
700                                 p_bind = NULL;
701                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
702                                         "osm_vendor_bind: ERR 3C16: "
703                                         "Error obtaining IB_MGT handle to SMI.\n");
704                                 goto Exit;
705                         }
706
707                         /* bind it */
708                         mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl);
709                         if (IB_MGT_OK != mgt_ret) {
710                                 free(p_bind);
711                                 p_bind = NULL;
712                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
713                                         "osm_vendor_bind: ERR 3C17: "
714                                         "Error binding IB_MGT handle to SM.\n");
715                                 goto Exit;
716                         }
717
718                         ib_mgt_hdl_p->smi_init = TRUE;
719
720                 }
721
722                 /* attach to this bind info */
723                 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl;
724                 ib_mgt_hdl_p->smi_p_bind = p_bind;
725
726                 /* now register the callback */
727                 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
728                                         &__osm_mtl_rcv_callback,
729                                         p_bind,
730                                         &__osm_mtl_send_callback,
731                                         p_bind,
732                                         IB_MGT_RCV_CB_MASK |
733                                         IB_MGT_SEND_CB_MASK);
734
735         } else {
736                 /*
737                  *  GSI CASE
738                  */
739
740                 if (ib_mgt_hdl_p->gsi_init == FALSE) {
741                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
742                                 "osm_vendor_bind: " "Binding to IB_MGT GSI\n");
743
744                         /* First we have to reg and get the handle for the mad */
745                         mgt_ret =
746                             IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI,
747                                               &(ib_mgt_hdl_p->gsi_mads_hdl));
748                         if (IB_MGT_OK != mgt_ret) {
749                                 free(p_bind);
750                                 p_bind = NULL;
751                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
752                                         "osm_vendor_bind: ERR 3C20: "
753                                         "Error obtaining IB_MGT handle to GSI.\n");
754                                 goto Exit;
755                         }
756
757                         /* bind it */
758                         mgt_ret =
759                             IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl,
760                                                   p_user_bind->mad_class);
761                         if (IB_MGT_OK != mgt_ret) {
762                                 free(p_bind);
763                                 p_bind = NULL;
764                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
765                                         "osm_vendor_bind: ERR 3C22: "
766                                         "Error binding IB_MGT handle to GSI.\n");
767                                 goto Exit;
768                         }
769
770                         ib_mgt_hdl_p->gsi_init = TRUE;
771
772                         /* attach to this bind info */
773                         p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
774
775                         /* now register the callback */
776                         mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
777                                                 &__osm_mtl_rcv_callback,
778                                                 p_bind,
779                                                 &__osm_mtl_send_callback,
780                                                 p_bind,
781                                                 IB_MGT_RCV_CB_MASK |
782                                                 IB_MGT_SEND_CB_MASK);
783
784                 } else {
785                         /* we can use the existing handle */
786                         p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
787                         mgt_ret = IB_MGT_OK;
788                 }
789
790         }
791
792         if (IB_MGT_OK != mgt_ret) {
793                 free(p_bind);
794                 p_bind = NULL;
795                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
796                         "osm_vendor_bind: ERR 3C23: "
797                         "Error binding IB_MGT CB (%d).\n", mgt_ret);
798                 goto Exit;
799         }
800
801         /* HACK: Do we need to initialize an address vector ???? */
802
803 Exit:
804         OSM_LOG_EXIT(p_vend->p_log);
805         return ((osm_bind_handle_t) p_bind);
806 }
807
808 /**********************************************************************
809 Get a mad from the lower level.
810 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
811 **********************************************************************/
812 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
813                          IN const uint32_t mad_size,
814                          IN osm_vend_wrap_t * const p_vw)
815 {
816         ib_mad_t *mad_p;
817         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
818         osm_vendor_t *p_vend = p_bind->p_vend;
819
820         OSM_LOG_ENTER(p_vend->p_log);
821
822         CL_ASSERT(p_vw);
823         /* HACK: We know we can not send through IB_MGT */
824         CL_ASSERT(mad_size <= MAD_BLOCK_SIZE);
825
826         /* IB_MGT assumes it is 256 - we must follow */
827         p_vw->size = MAD_BLOCK_SIZE;
828
829         /* allocate it */
830         mad_p = (ib_mad_t *) malloc(p_vw->size);
831         if (mad_p == NULL) {
832                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
833                         "osm_vendor_get: ERR 3C24: "
834                         "Error Obtaining MAD buffer.\n");
835                 goto Exit;
836         }
837
838         memset(mad_p, 0, p_vw->size);
839
840         /* track locally */
841         p_vw->mad_buf_p = mad_p;
842         p_vw->h_bind = h_bind;
843         p_vw->p_resp_madw = NULL;
844
845         if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
846                 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
847                         "osm_vendor_get: "
848                         "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size);
849         }
850
851 Exit:
852         OSM_LOG_EXIT(p_vend->p_log);
853         return (mad_p);
854 }
855
856 /**********************************************************************
857  * Return a MAD by providing it's wrapper object.
858  **********************************************************************/
859 void
860 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
861 {
862         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
863         osm_vendor_t *p_vend = p_bind->p_vend;
864         osm_madw_t *p_madw;
865
866         OSM_LOG_ENTER(p_vend->p_log);
867
868         CL_ASSERT(p_vw);
869         CL_ASSERT(p_vw->mad_buf_p);
870
871         if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
872                 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
873                         "osm_vendor_put: " "Retiring MAD %p.\n",
874                         p_vw->mad_buf_p);
875         }
876
877         /*
878          * We moved the removal of the transaction to immediatly after
879          * it was looked up.
880          */
881
882         /* free the mad but the wrapper is part of the madw object */
883         free(p_vw->mad_buf_p);
884         p_vw->mad_buf_p = NULL;
885         p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
886         p_madw->p_mad = NULL;
887
888         OSM_LOG_EXIT(p_vend->p_log);
889 }
890
891 /**********************************************************************
892 Actually Send a MAD
893
894 This is for internal use by osm_vendor_send and the transaction mgr
895 retry too.
896 **********************************************************************/
897 ib_api_status_t
898 osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
899 {
900         osm_vendor_t *const p_vend = p_bind->p_vend;
901         osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
902         osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
903         ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
904         ib_api_status_t status;
905         IB_MGT_ret_t mgt_res;
906         IB_ud_av_t av;
907         uint64_t wrid;
908         uint32_t qpn;
909
910         OSM_LOG_ENTER(p_vend->p_log);
911
912         /*
913          * For all sends other than directed route SM MADs,
914          * acquire an address vector for the destination.
915          */
916         if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
917                 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr,
918                                                       p_mad->mgmt_class ==
919                                                       IB_MCLASS_SUBN_LID, &av);
920         } else {
921                 /* is a directed route - we need to construct a permissive address */
922                 memset(&av, 0, sizeof(av));
923                 /* we do not need port number since it is part of the mad_hndl */
924                 av.dlid = IB_LID_PERMISSIVE;
925         }
926
927         wrid = __osm_set_wrid_by_p_madw(p_madw);
928
929         /* send it */
930         if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
931             (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
932
933                 /* SMI CASE */
934                 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
935                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
936                                 "osm_mtl_send_mad: "
937                                 "av.dlid 0x%X, "
938                                 "av.static_rate %d, "
939                                 "av.path_bits %d.\n",
940                                 cl_ntoh16(av.dlid), av.static_rate,
941                                 av.src_path_bits);
942                 }
943
944                 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad,      /*  actual payload */
945                                           &av,  /*  address vector */
946                                           wrid, /*  casting the mad wrapper pointer for err cb */
947                                           p_vend->timeout);
948
949         } else {
950                 /* GSI CASE - Support Remote QP */
951                 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
952                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
953                                 "osm_mtl_send_mad: "
954                                 "av.dlid 0x%X, av.static_rate %d, "
955                                 "av.path_bits %d, remote qp: 0x%06X \n",
956                                 av.dlid,
957                                 av.static_rate,
958                                 av.src_path_bits,
959                                 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
960                             );
961                 }
962
963                 /* IBMGT have a bug sending to a QP not 1 -
964                    the QPN must be in network order except when it qpn 1 ... */
965                 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
966
967                 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad,        /*  actual payload */
968                                                 &av,    /* address vector */
969                                                 wrid,   /* casting the mad wrapper pointer for err cb */
970                                                 p_vend->timeout, qpn);
971         }
972
973         if (mgt_res != IB_MGT_OK) {
974                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
975                         "osm_mtl_send_mad: ERR 3C26: "
976                         "Error sending mad (%d).\n", mgt_res);
977                 if (p_vw->p_resp_madw)
978                         osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
979                 status = IB_ERROR;
980                 goto Exit;
981         }
982
983         status = IB_SUCCESS;
984
985 Exit:
986         OSM_LOG_EXIT(p_vend->p_log);
987         return (status);
988 }
989
990 /**********************************************************************
991 Send a MAD through.
992
993 What is unclear to me is the need for the setting of all the MAD Wrapper
994 fields. Seems like the OSM uses these values during it's processing...
995 **********************************************************************/
996 ib_api_status_t
997 osm_vendor_send(IN osm_bind_handle_t h_bind,
998                 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
999 {
1000         osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind;
1001         osm_vendor_t *const p_vend = p_bind->p_vend;
1002         osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1003         ib_api_status_t status;
1004
1005         OSM_LOG_ENTER(p_vend->p_log);
1006
1007         /*
1008          * If a response is expected to this MAD, then preallocate
1009          * a mad wrapper to contain the wire MAD received in the
1010          * response.  Allocating a wrapper here allows for easier
1011          * failure paths than after we already received the wire mad.
1012          */
1013         if (resp_expected == TRUE) {
1014                 /* we track it in the vendor wrapper */
1015                 p_vw->p_resp_madw =
1016                     osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
1017                 if (p_vw->p_resp_madw == NULL) {
1018                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
1019                                 "osm_vendor_send: ERR 3C27: "
1020                                 "Unable to allocate MAD wrapper.\n");
1021                         status = IB_INSUFFICIENT_RESOURCES;
1022                         goto Exit;
1023                 }
1024
1025                 /* put some minimal info on that wrapper */
1026                 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
1027
1028                 /* we also want to track it in the TID based map */
1029                 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t)
1030                                                          p_bind, p_madw);
1031                 if (status != IB_SUCCESS) {
1032                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
1033                                 "osm_vendor_send: ERR 3C25: "
1034                                 "Error inserting request madw by TID (%d).\n",
1035                                 status);
1036                 }
1037
1038         } else
1039                 p_vw->p_resp_madw = NULL;
1040
1041         /* do the actual send */
1042         status = osm_mtl_send_mad(p_bind, p_madw);
1043
1044 Exit:
1045         OSM_LOG_EXIT(p_vend->p_log);
1046         return (status);
1047 }
1048
1049 /**********************************************************************
1050  * the idea here is to change the content of the bind such that it
1051  * will hold the local address used for sending directed route by the SMA.
1052  **********************************************************************/
1053 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1054 {
1055         osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend;
1056
1057         OSM_LOG_ENTER(p_vend->p_log);
1058
1059         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
1060                 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
1061
1062         OSM_LOG_EXIT(p_vend->p_log);
1063
1064         return (IB_SUCCESS);
1065 }
1066
1067 /**********************************************************************
1068  **********************************************************************/
1069 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1070 {
1071         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
1072         osm_vendor_t *p_vend = p_bind->p_vend;
1073         VAPI_ret_t status;
1074         VAPI_hca_attr_t attr_mod;
1075         VAPI_hca_attr_mask_t attr_mask;
1076
1077         OSM_LOG_ENTER(p_vend->p_log);
1078
1079         memset(&attr_mod, 0, sizeof(attr_mod));
1080         memset(&attr_mask, 0, sizeof(attr_mask));
1081
1082         attr_mod.is_sm = is_sm_val;
1083         attr_mask = HCA_ATTR_IS_SM;
1084
1085         status =
1086             VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
1087                                  &attr_mask);
1088         if (status != VAPI_OK) {
1089                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1090                         "osm_vendor_set_sm: ERR 3C28: "
1091                         "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
1092                         is_sm_val, status);
1093         }
1094
1095         OSM_LOG_EXIT(p_vend->p_log);
1096 }
1097
1098 /**********************************************************************
1099  **********************************************************************/
1100 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1101 {
1102
1103 }
1104
1105 #endif                          /* OSM_VENDOR_INTF_TEST */