]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - contrib/ofed/opensm/libvendor/osm_vendor_mtl.c
Update ncurses to 20200118
[FreeBSD/FreeBSD.git] / contrib / ofed / opensm / libvendor / osm_vendor_mtl.c
1 /*
2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #if HAVE_CONFIG_H
37 #  include <config.h>
38 #endif                          /* HAVE_CONFIG_H */
39
40 #ifdef OSM_VENDOR_INTF_MTL
41
42 #include <stdlib.h>
43 #include <string.h>
44 #include <opensm/osm_helper.h>
45 #include <opensm/osm_log.h>
46 /* HACK - I do not know how to prevent complib from loading kernel H files */
47 #undef __init
48 #include <vendor/osm_vendor_mtl.h>
49 #include <vendor/osm_vendor_api.h>
50 #include <opensm/osm_subnet.h>
51 #include <opensm/osm_opensm.h>
52 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
53 #include <vendor/osm_mtl_bind.h>
54
55 /*
56   Since a race can accure on requests. Meaning - a response is received before
57   the send_callback is called - we will save both the madw_p and the fact
58   whether or not it is a response. A race can occure only on requests that did
59   not fail, and then the madw_p will be put back in the pool before the callback.
60 */
61 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
62 {
63         uint64_t wrid = 0;
64
65         CL_ASSERT(p_madw->p_mad);
66
67         memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
68         wrid = (wrid << 1) |
69             ib_mad_is_response(p_madw->p_mad);
70         return wrid;
71 }
72
73 void
74 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
75                                   OUT uint8_t * is_resp,
76                                   OUT osm_madw_t ** pp_madw)
77 {
78         *is_resp = wrid & 0x0000000000000001;
79         wrid = wrid >> 1;
80         memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
81 }
82
83 /**********************************************************************
84  * IB_MGT to OSM ADDRESS VECTOR
85  **********************************************************************/
86 void
87 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
88                                           IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
89                                           IN uint8_t is_smi,
90                                           OUT osm_mad_addr_t * p_mad_addr)
91 {
92         /*  p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
93         p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
94         p_mad_addr->static_rate = 0;    /*  HACK - we do not  know the rate ! */
95         p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
96         if (is_smi) {
97                 /* SMI */
98                 p_mad_addr->addr_type.smi.source_lid =
99                     cl_hton16(p_rcv_desc->remote_lid);
100                 p_mad_addr->addr_type.smi.port_num = 99;        /*  HACK - if used - should fail */
101         } else {
102                 /* GSI */
103                 /* seems to me there is a IBMGT bug reversing the QPN ... */
104                 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
105                 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
106
107                 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
108                 /*  we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
109                 /*  the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
110                 /*  the full PKey table - than go by the index. */
111                 /*  since this does not seem reasonable to me I simply use the default */
112                 /*  There is a TAVOR limitation that only one P_KEY is supported per  */
113                 /*  QP - so QP1 must use IB_DEFAULT_PKEY */
114                 p_mad_addr->addr_type.gsi.pkey_ix = 0;
115                 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
116
117                 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
118                 /* copy the GRH data if relevant */
119                 if (p_mad_addr->addr_type.gsi.global_route) {
120                         p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
121                             ib_grh_set_ver_class_flow(p_rcv_desc->grh.
122                                                       IP_version,
123                                                       p_rcv_desc->grh.
124                                                       traffic_class,
125                                                       p_rcv_desc->grh.
126                                                       flow_label);
127                         p_mad_addr->addr_type.gsi.grh_info.hop_limit =
128                             p_rcv_desc->grh.hop_limit;
129                         memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
130                                &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
131                         memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
132                                p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
133                 }
134         }
135 }
136
137 /**********************************************************************
138  * OSM ADDR VECTOR TO IB_MGT
139  **********************************************************************/
140 void
141 __osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,
142                                       IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
143 {
144
145         /* For global destination or Multicast address: */
146         u_int8_t ver;
147
148         memset(p_av, 0, sizeof(IB_ud_av_t));
149
150         p_av->src_path_bits = p_mad_addr->path_bits;
151         p_av->static_rate = p_mad_addr->static_rate;
152         p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
153
154         if (is_smi) {
155                 p_av->sl = 0;   /*  Just to note we use 0 here. */
156         } else {
157                 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
158                 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
159
160                 if (p_mad_addr->addr_type.gsi.global_route) {
161                         ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
162                                                   grh_info.ver_class_flow, &ver,
163                                                   &p_av->traffic_class,
164                                                   &p_av->flow_label);
165                         p_av->hop_limit =
166                             p_mad_addr->addr_type.gsi.grh_info.hop_limit;
167                         p_av->sgid_index = 0;   /*  we always use source GID 0 */
168                         memcpy(&p_av->dgid,
169                                &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
170                                sizeof(ib_net64_t));
171
172                 }
173         }
174 }
175
176 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
177 {
178         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
179         osm_vendor_t *p_vend = p_bind->p_vend;
180         VAPI_ret_t status;
181         VAPI_hca_attr_t attr_mod;
182         VAPI_hca_attr_mask_t attr_mask;
183
184         OSM_LOG_ENTER(p_vend->p_log);
185
186         memset(&attr_mod, 0, sizeof(attr_mod));
187         memset(&attr_mask, 0, sizeof(attr_mask));
188
189         attr_mod.is_sm = FALSE;
190         attr_mask = HCA_ATTR_IS_SM;
191
192         status =
193             VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
194                                  &attr_mask);
195         if (status != VAPI_OK) {
196                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
197                         "__osm_vendor_clear_sm: ERR 3C21: "
198                         "Unable set 'IS_SM' bit in port attributes (%d).\n",
199                         status);
200         }
201
202         OSM_LOG_EXIT(p_vend->p_log);
203 }
204
205 /**********************************************************************
206  * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
207  **********************************************************************/
208 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
209 {
210         memset(p_vend, 0, sizeof(*p_vend));
211 }
212
213 /**********************************************************************
214  * DEALOCATE osm_vendor_t
215  **********************************************************************/
216 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
217 {
218         osm_vendor_mgt_bind_t *vendor_mgt_bind_p;
219         IB_MGT_ret_t mgt_ret;
220         OSM_LOG_ENTER(p_vend->p_log);
221
222         if (p_vend->h_al != NULL) {
223                 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
224                 if (vendor_mgt_bind_p->gsi_init) {
225
226                         /* un register the class */
227                         /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */
228                         mgt_ret =
229                             IB_MGT_unbind_gsi_class(vendor_mgt_bind_p->
230                                                     gsi_mads_hdl,
231                                                     IB_MCLASS_SUBN_ADM);
232                         if (mgt_ret != IB_MGT_OK) {
233                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
234                                         "osm_vendor_destroy: ERR 3C03: "
235                                         "Fail to unbind the SA class.\n");
236                         }
237
238                         /* un bind the handle */
239                         if (IB_MGT_release_handle
240                             (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) {
241                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
242                                         "osm_vendor_destroy: ERR 3C02: "
243                                         "Fail to unbind the SA GSI handle.\n");
244                         }
245                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
246                                 "osm_vendor_destroy: DBG 1002: "
247                                 "Unbind the GSI handles.\n");
248                 }
249                 if (vendor_mgt_bind_p->smi_init) {
250                         /* first - clear the IS_SM in the capability mask */
251                         __osm_vendor_clear_sm((osm_bind_handle_t)
252                                               (vendor_mgt_bind_p->smi_p_bind));
253
254                         /* un register the class */
255                         mgt_ret =
256                             IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl);
257                         if (mgt_ret != IB_MGT_OK) {
258                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
259                                         "osm_vendor_destroy: ERR 3C04: "
260                                         "Fail to unbind the SM class.\n");
261                         }
262
263                         /* un bind the handle */
264                         if (IB_MGT_release_handle
265                             (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) {
266                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
267                                         "osm_vendor_destroy: ERR 3C05: "
268                                         "Fail to unbind the SMI handle.\n");
269                         }
270                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
271                                 "osm_vendor_destroy: DBG 1003: "
272                                 "Unbind the SMI handles.\n");
273
274                 }
275         }
276         osm_transaction_mgr_destroy(p_vend);
277         /*  __osm_mtl_destroy_tid_mad_map( p_vend ); */
278         OSM_LOG_EXIT(p_vend->p_log);
279 }
280
281 /**********************************************************************
282 DEALLOCATE A POINTER TO osm_vendor_t
283 **********************************************************************/
284 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
285 {
286         CL_ASSERT(pp_vend);
287
288         osm_vendor_destroy(*pp_vend);
289         free(*pp_vend);
290         *pp_vend = NULL;
291 }
292
293 /**********************************************************************
294  * This proc actuall binds the handle to the lower level.
295  *
296  * We might have here as a result a casting of our struct to the ib_al_handle_t
297  *
298  * Q: Do we need 2 of those - one for MSI and one for GSI ?
299  * A: Yes! We should be able to do the SA too. So we need a struct!
300  *
301  **********************************************************************/
302
303 ib_api_status_t
304 osm_vendor_init(IN osm_vendor_t * const p_vend,
305                 IN osm_log_t * const p_log, IN const uint32_t timeout)
306 {
307         osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
308         ib_api_status_t status = IB_SUCCESS;
309
310         OSM_LOG_ENTER(p_log);
311
312         p_vend->p_log = p_log;
313
314         /*
315          * HACK: We need no handle. Assuming the driver is up.
316          */
317         ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *)
318             malloc(sizeof(osm_vendor_mgt_bind_t));
319         if (ib_mgt_hdl_p == NULL) {
320                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
321                         "osm_vendor_init: ERR 3C06: "
322                         "Fail to allocate vendor mgt handle.\n");
323                 goto Exit;
324         }
325
326         ib_mgt_hdl_p->smi_init = FALSE;
327         ib_mgt_hdl_p->gsi_init = FALSE;
328         /* cast it into the ib_al_handle_t h_al */
329         p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p;
330         p_vend->p_transaction_mgr = NULL;
331         osm_transaction_mgr_init(p_vend);
332         /*  p_vend->madw_by_tid_map_p = NULL; */
333         /*  __osm_mtl_init_tid_mad_map( p_vend ); */
334         p_vend->timeout = timeout;
335
336 Exit:
337         OSM_LOG_EXIT(p_log);
338         return (status);
339 }
340
341 /**********************************************************************
342  *  Create and Initialize osm_vendor_t Object
343  **********************************************************************/
344 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
345                              IN const uint32_t timeout)
346 {
347         ib_api_status_t status;
348         osm_vendor_t *p_vend;
349
350         OSM_LOG_ENTER(p_log);
351
352         CL_ASSERT(p_log);
353
354         p_vend = malloc(sizeof(*p_vend));
355         if (p_vend != NULL) {
356                 memset(p_vend, 0, sizeof(*p_vend));
357                 status = osm_vendor_init(p_vend, p_log, timeout);
358                 if (status != IB_SUCCESS) {
359                         osm_vendor_delete(&p_vend);
360                 }
361         } else {
362                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
363                         "osm_vendor_new: ERR 3C07: "
364                         "Fail to allocate vendor object.\n");
365         }
366
367         OSM_LOG_EXIT(p_log);
368         return (p_vend);
369 }
370
371 /**********************************************************************
372  * IB_MGT RCV callback
373  *
374  **********************************************************************/
375 void
376 __osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,
377                        IN void *private_ctx_p,
378                        IN void *payload_p,
379                        IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
380 {
381         IB_MGT_ret_t status;
382         osm_mtl_bind_info_t *bind_info_p = private_ctx_p;
383         osm_madw_t *req_madw_p = NULL;
384         osm_madw_t *madw_p;
385         osm_vend_wrap_t *p_new_vw;
386         osm_mad_addr_t mad_addr;
387         ib_mad_t *mad_buf_p;
388         osm_log_t *const p_log = bind_info_p->p_vend->p_log;
389
390         OSM_LOG_ENTER(p_log);
391
392         /* if it is a response MAD we mustbe able to get the request */
393         if (ib_mad_is_response((ib_mad_t *) payload_p)) {
394                 /* can we find a matching madw by this payload TID */
395                 status =
396                     osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend,
397                                                          (ib_mad_t *) payload_p,
398                                                          &req_madw_p);
399                 if (status != IB_MGT_OK) {
400                         osm_log(p_log, OSM_LOG_ERROR,
401                                 "__osm_mtl_rcv_callback: ERR 3C08: "
402                                 "Error obtaining request madw by TID (%d).\n",
403                                 status);
404                         req_madw_p = NULL;
405                 }
406
407                 if (req_madw_p == NULL) {
408                         osm_log(p_log, OSM_LOG_ERROR,
409                                 "__osm_mtl_rcv_callback: ERR 3C09:  "
410                                 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n",
411                                 ((ib_mad_t *) payload_p)->method,
412                                 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id)
413
414                             );
415                         goto Exit;
416                 }
417         }
418
419         /* do we have a request ??? */
420         if (req_madw_p == NULL) {
421
422                 /* first arrange an address */
423                 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend,
424                                                           rcv_remote_info_p,
425                                                           (((ib_mad_t *)
426                                                             payload_p)->
427                                                            mgmt_class ==
428                                                            IB_MCLASS_SUBN_LID)
429                                                           || (((ib_mad_t *)
430                                                                payload_p)->
431                                                               mgmt_class ==
432                                                               IB_MCLASS_SUBN_DIR),
433                                                           &mad_addr);
434
435                 osm_log(p_log, OSM_LOG_ERROR,
436                         "__osm_mtl_rcv_callback: : "
437                         "Received MAD from QP:%X.\n",
438                         cl_ntoh32(mad_addr.addr_type.gsi.remote_qp)
439                     );
440
441                 /* if not - get new osm_madw and arrange it. */
442                 /* create the new madw in the pool */
443                 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool,
444                                           (osm_bind_handle_t) bind_info_p,
445                                           MAD_BLOCK_SIZE, &mad_addr);
446                 if (madw_p == NULL) {
447                         osm_log(p_log, OSM_LOG_ERROR,
448                                 "__osm_mtl_rcv_callback: ERR 3C10: "
449                                 "Error request for a new madw.\n");
450                         goto Exit;
451                 }
452                 /* HACK: we cust to avoid the const ??? */
453                 mad_buf_p = (void *)madw_p->p_mad;
454         } else {
455                 /* we have the madw defined during the send and stored in the vend_wrap */
456                 /* we need to make sure the wrapper is correctly init there */
457                 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0);
458                 madw_p = req_madw_p->vend_wrap.p_resp_madw;
459
460                 /* HACK: we do not Support RMPP */
461                 CL_ASSERT(madw_p->h_bind);
462                 mad_buf_p =
463                     osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE,
464                                    &madw_p->vend_wrap);
465
466                 if (mad_buf_p == NULL) {
467                         osm_log(p_log, OSM_LOG_ERROR,
468                                 "__osm_mtl_rcv_callback: ERR 3C11: "
469                                 "Unable to acquire wire MAD.\n");
470
471                         goto Exit;
472                 }
473
474                 /*
475                    Finally, attach the wire MAD to this wrapper.
476                  */
477                 osm_madw_set_mad(madw_p, mad_buf_p);
478
479                 /* also we need to handle the size of the mad since we did not init ... */
480                 madw_p->mad_size = MAD_BLOCK_SIZE;
481         }
482
483         /* init some fields of the vendor wrapper */
484         p_new_vw = osm_madw_get_vend_ptr(madw_p);
485         p_new_vw->h_bind = bind_info_p;
486         p_new_vw->size = MAD_BLOCK_SIZE;
487         p_new_vw->p_resp_madw = NULL;
488         p_new_vw->mad_buf_p = mad_buf_p;
489
490         /* HACK: We do not support RMPP in receiving MADS */
491         memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE);
492
493         /* attach the buffer to the wrapper */
494         madw_p->p_mad = mad_buf_p;
495
496         /* we can also make sure we marked the size and bind on the returned madw */
497         madw_p->h_bind = p_new_vw->h_bind;
498
499         /* call the CB */
500         (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context,
501                                       req_madw_p);
502
503 Exit:
504         OSM_LOG_EXIT(p_log);
505 }
506
507 /**********************************************************************
508  * IB_MGT Send callback : invoked after each send
509  *
510  **********************************************************************/
511 void
512 __osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,
513                         IN u_int64_t wrid,
514                         IN IB_comp_status_t status, IN void *private_ctx_p)
515 {
516         osm_madw_t *madw_p;
517         osm_mtl_bind_info_t *bind_info_p =
518             (osm_mtl_bind_info_t *) private_ctx_p;
519         osm_log_t *const p_log = bind_info_p->p_vend->p_log;
520         osm_vend_wrap_t *p_vw;
521         uint8_t is_resp;
522
523         OSM_LOG_ENTER(p_log);
524
525         /* obtain the madp from the wrid */
526         __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p);
527
528         osm_log(p_log, OSM_LOG_DEBUG,
529                 "__osm_mtl_send_callback: INFO 1008: "
530                 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
531
532         /* we need to handle requests and responses differently */
533         if (is_resp) {
534                 if (status != IB_COMP_SUCCESS) {
535                         osm_log(p_log, OSM_LOG_ERROR,
536                                 "__osm_mtl_send_callback: ERR 3C12: "
537                                 "Error Sending Response MADW:%p.\n", madw_p);
538                 } else {
539                         osm_log(p_log, OSM_LOG_DEBUG,
540                                 "__osm_mtl_send_callback: DBG 1008: "
541                                 "Completed Sending Response MADW:%p.\n",
542                                 madw_p);
543                 }
544
545                 /* if we are a response - we need to clean it up */
546                 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
547         } else {
548
549                 /* this call back is invoked on completion of send - error or not */
550                 if (status != IB_COMP_SUCCESS) {
551
552                         osm_log(p_log, OSM_LOG_ERROR,
553                                 "__osm_mtl_send_callback: ERR 3C13: "
554                                 "Received an Error from IB_MGT Send (%d).\n",
555                                 status);
556
557                         p_vw = osm_madw_get_vend_ptr(madw_p);
558                         CL_ASSERT(p_vw);
559
560                         /*
561                            Return any wrappers to the pool that may have been
562                            pre-emptively allocated to handle a receive.
563                          */
564                         if (p_vw->p_resp_madw) {
565                                 osm_mad_pool_put(bind_info_p->p_osm_pool,
566                                                  p_vw->p_resp_madw);
567                                 p_vw->p_resp_madw = NULL;
568                         }
569
570                         /* invoke the CB */
571                         (*bind_info_p->send_err_callback) (bind_info_p->
572                                                            client_context,
573                                                            madw_p);
574                 } else {
575                         /* successful request send - do nothing - the response will need the
576                            out mad */
577                         osm_log(p_log, OSM_LOG_DEBUG,
578                                 "__osm_mtl_send_callback: DBG 1008: "
579                                 "Completed Sending Request MADW:%p.\n", madw_p);
580                 }
581         }
582
583         OSM_LOG_EXIT(p_log);
584 }
585
586 /**********************************************************************
587  * BINDs a callback (rcv and send error) for a given class and method
588  * defined by the given:  osm_bind_info_t
589  **********************************************************************/
590 osm_bind_handle_t
591 osm_vendor_bind(IN osm_vendor_t * const p_vend,
592                 IN osm_bind_info_t * const p_user_bind,
593                 IN osm_mad_pool_t * const p_mad_pool,
594                 IN osm_vend_mad_recv_callback_t mad_recv_callback,
595                 IN osm_vend_mad_send_err_callback_t send_err_callback,
596                 IN void *context)
597 {
598         ib_net64_t port_guid;
599         osm_mtl_bind_info_t *p_bind = NULL;
600         VAPI_hca_hndl_t hca_hndl;
601         VAPI_hca_id_t hca_id;
602         IB_MGT_mad_type_t mad_type;
603         uint32_t port_num;
604         osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
605         IB_MGT_ret_t mgt_ret;
606
607         OSM_LOG_ENTER(p_vend->p_log);
608
609         CL_ASSERT(p_user_bind);
610         CL_ASSERT(p_mad_pool);
611         CL_ASSERT(mad_recv_callback);
612         CL_ASSERT(send_err_callback);
613
614         /* cast back the AL handle to vendor mgt bind */
615         ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
616
617         port_guid = p_user_bind->port_guid;
618
619         osm_log(p_vend->p_log, OSM_LOG_INFO,
620                 "osm_vendor_bind: "
621                 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
622
623         /* obtain the hca name and port num from the guid */
624         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
625                 "osm_vendor_bind: "
626                 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
627                 port_guid);
628
629         mgt_ret =
630             osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
631                                             &hca_id, &port_num);
632         if (mgt_ret != IB_MGT_OK) {
633                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
634                         "osm_vendor_bind: ERR 3C14: "
635                         "Unable to obtain CA and port (%d).\n");
636                 goto Exit;
637         }
638
639         /* create the bind object tracking this binding */
640         p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t));
641         memset(p_bind, 0, sizeof(osm_mtl_bind_info_t));
642         if (p_bind == NULL) {
643                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
644                         "osm_vendor_bind: ERR 3C15: "
645                         "Unable to allocate internal bind object.\n");
646                 goto Exit;
647         }
648
649         /* track this bind request info */
650         memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t));
651         p_bind->port_num = port_num;
652         p_bind->p_vend = p_vend;
653         p_bind->client_context = context;
654         p_bind->rcv_callback = mad_recv_callback;
655         p_bind->send_err_callback = send_err_callback;
656         p_bind->p_osm_pool = p_mad_pool;
657
658         CL_ASSERT(p_bind->port_num);
659
660         /*
661          * Get the proper CLASS
662          */
663
664         switch (p_user_bind->mad_class) {
665         case IB_MCLASS_SUBN_LID:
666         case IB_MCLASS_SUBN_DIR:
667                 mad_type = IB_MGT_SMI;
668                 break;
669
670         case IB_MCLASS_SUBN_ADM:
671         default:
672                 mad_type = IB_MGT_GSI;
673                 break;
674         }
675
676         /* we split here - based on the type of MADS GSI / SMI */
677         /* HACK: we only support one class registration per SMI/GSI !!! */
678         if (mad_type == IB_MGT_SMI) {
679                 /*
680                  *  SMI CASE
681                  */
682
683                 /* we do not need to bind the handle if already available */
684                 if (ib_mgt_hdl_p->smi_init == FALSE) {
685
686                         /* First we have to reg and get the handle for the mad */
687                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
688                                 "osm_vendor_bind: "
689                                 "Binding to IB_MGT SMI of %s port %u\n", hca_id,
690                                 port_num);
691
692                         mgt_ret =
693                             IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI,
694                                               &(ib_mgt_hdl_p->smi_mads_hdl));
695                         if (IB_MGT_OK != mgt_ret) {
696                                 free(p_bind);
697                                 p_bind = NULL;
698                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
699                                         "osm_vendor_bind: ERR 3C16: "
700                                         "Error obtaining IB_MGT handle to SMI.\n");
701                                 goto Exit;
702                         }
703
704                         /* bind it */
705                         mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl);
706                         if (IB_MGT_OK != mgt_ret) {
707                                 free(p_bind);
708                                 p_bind = NULL;
709                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
710                                         "osm_vendor_bind: ERR 3C17: "
711                                         "Error binding IB_MGT handle to SM.\n");
712                                 goto Exit;
713                         }
714
715                         ib_mgt_hdl_p->smi_init = TRUE;
716
717                 }
718
719                 /* attach to this bind info */
720                 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl;
721                 ib_mgt_hdl_p->smi_p_bind = p_bind;
722
723                 /* now register the callback */
724                 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
725                                         &__osm_mtl_rcv_callback,
726                                         p_bind,
727                                         &__osm_mtl_send_callback,
728                                         p_bind,
729                                         IB_MGT_RCV_CB_MASK |
730                                         IB_MGT_SEND_CB_MASK);
731
732         } else {
733                 /*
734                  *  GSI CASE
735                  */
736
737                 if (ib_mgt_hdl_p->gsi_init == FALSE) {
738                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
739                                 "osm_vendor_bind: " "Binding to IB_MGT GSI\n");
740
741                         /* First we have to reg and get the handle for the mad */
742                         mgt_ret =
743                             IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI,
744                                               &(ib_mgt_hdl_p->gsi_mads_hdl));
745                         if (IB_MGT_OK != mgt_ret) {
746                                 free(p_bind);
747                                 p_bind = NULL;
748                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
749                                         "osm_vendor_bind: ERR 3C20: "
750                                         "Error obtaining IB_MGT handle to GSI.\n");
751                                 goto Exit;
752                         }
753
754                         /* bind it */
755                         mgt_ret =
756                             IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl,
757                                                   p_user_bind->mad_class);
758                         if (IB_MGT_OK != mgt_ret) {
759                                 free(p_bind);
760                                 p_bind = NULL;
761                                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
762                                         "osm_vendor_bind: ERR 3C22: "
763                                         "Error binding IB_MGT handle to GSI.\n");
764                                 goto Exit;
765                         }
766
767                         ib_mgt_hdl_p->gsi_init = TRUE;
768
769                         /* attach to this bind info */
770                         p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
771
772                         /* now register the callback */
773                         mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
774                                                 &__osm_mtl_rcv_callback,
775                                                 p_bind,
776                                                 &__osm_mtl_send_callback,
777                                                 p_bind,
778                                                 IB_MGT_RCV_CB_MASK |
779                                                 IB_MGT_SEND_CB_MASK);
780
781                 } else {
782                         /* we can use the existing handle */
783                         p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
784                         mgt_ret = IB_MGT_OK;
785                 }
786
787         }
788
789         if (IB_MGT_OK != mgt_ret) {
790                 free(p_bind);
791                 p_bind = NULL;
792                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
793                         "osm_vendor_bind: ERR 3C23: "
794                         "Error binding IB_MGT CB (%d).\n", mgt_ret);
795                 goto Exit;
796         }
797
798         /* HACK: Do we need to initialize an address vector ???? */
799
800 Exit:
801         OSM_LOG_EXIT(p_vend->p_log);
802         return ((osm_bind_handle_t) p_bind);
803 }
804
805 /**********************************************************************
806 Get a mad from the lower level.
807 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
808 **********************************************************************/
809 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
810                          IN const uint32_t mad_size,
811                          IN osm_vend_wrap_t * const p_vw)
812 {
813         ib_mad_t *mad_p;
814         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
815         osm_vendor_t *p_vend = p_bind->p_vend;
816
817         OSM_LOG_ENTER(p_vend->p_log);
818
819         CL_ASSERT(p_vw);
820         /* HACK: We know we can not send through IB_MGT */
821         CL_ASSERT(mad_size <= MAD_BLOCK_SIZE);
822
823         /* IB_MGT assumes it is 256 - we must follow */
824         p_vw->size = MAD_BLOCK_SIZE;
825
826         /* allocate it */
827         mad_p = (ib_mad_t *) malloc(p_vw->size);
828         if (mad_p == NULL) {
829                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
830                         "osm_vendor_get: ERR 3C24: "
831                         "Error Obtaining MAD buffer.\n");
832                 goto Exit;
833         }
834
835         memset(mad_p, 0, p_vw->size);
836
837         /* track locally */
838         p_vw->mad_buf_p = mad_p;
839         p_vw->h_bind = h_bind;
840         p_vw->p_resp_madw = NULL;
841
842         if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
843                 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
844                         "osm_vendor_get: "
845                         "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size);
846         }
847
848 Exit:
849         OSM_LOG_EXIT(p_vend->p_log);
850         return (mad_p);
851 }
852
853 /**********************************************************************
854  * Return a MAD by providing it's wrapper object.
855  **********************************************************************/
856 void
857 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
858 {
859         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
860         osm_vendor_t *p_vend = p_bind->p_vend;
861         osm_madw_t *p_madw;
862
863         OSM_LOG_ENTER(p_vend->p_log);
864
865         CL_ASSERT(p_vw);
866         CL_ASSERT(p_vw->mad_buf_p);
867
868         if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
869                 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
870                         "osm_vendor_put: " "Retiring MAD %p.\n",
871                         p_vw->mad_buf_p);
872         }
873
874         /*
875          * We moved the removal of the transaction to immediatly after
876          * it was looked up.
877          */
878
879         /* free the mad but the wrapper is part of the madw object */
880         free(p_vw->mad_buf_p);
881         p_vw->mad_buf_p = NULL;
882         p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
883         p_madw->p_mad = NULL;
884
885         OSM_LOG_EXIT(p_vend->p_log);
886 }
887
888 /**********************************************************************
889 Actually Send a MAD
890
891 This is for internal use by osm_vendor_send and the transaction mgr
892 retry too.
893 **********************************************************************/
894 ib_api_status_t
895 osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
896 {
897         osm_vendor_t *const p_vend = p_bind->p_vend;
898         osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
899         osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
900         ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
901         ib_api_status_t status;
902         IB_MGT_ret_t mgt_res;
903         IB_ud_av_t av;
904         uint64_t wrid;
905         uint32_t qpn;
906
907         OSM_LOG_ENTER(p_vend->p_log);
908
909         /*
910          * For all sends other than directed route SM MADs,
911          * acquire an address vector for the destination.
912          */
913         if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
914                 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr,
915                                                       p_mad->mgmt_class ==
916                                                       IB_MCLASS_SUBN_LID, &av);
917         } else {
918                 /* is a directed route - we need to construct a permissive address */
919                 memset(&av, 0, sizeof(av));
920                 /* we do not need port number since it is part of the mad_hndl */
921                 av.dlid = IB_LID_PERMISSIVE;
922         }
923
924         wrid = __osm_set_wrid_by_p_madw(p_madw);
925
926         /* send it */
927         if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
928             (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
929
930                 /* SMI CASE */
931                 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
932                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
933                                 "osm_mtl_send_mad: "
934                                 "av.dlid 0x%X, "
935                                 "av.static_rate %d, "
936                                 "av.path_bits %d.\n",
937                                 cl_ntoh16(av.dlid), av.static_rate,
938                                 av.src_path_bits);
939                 }
940
941                 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad,      /*  actual payload */
942                                           &av,  /*  address vector */
943                                           wrid, /*  casting the mad wrapper pointer for err cb */
944                                           p_vend->timeout);
945
946         } else {
947                 /* GSI CASE - Support Remote QP */
948                 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
949                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
950                                 "osm_mtl_send_mad: "
951                                 "av.dlid 0x%X, av.static_rate %d, "
952                                 "av.path_bits %d, remote qp: 0x%06X \n",
953                                 av.dlid,
954                                 av.static_rate,
955                                 av.src_path_bits,
956                                 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
957                             );
958                 }
959
960                 /* IBMGT have a bug sending to a QP not 1 -
961                    the QPN must be in network order except when it qpn 1 ... */
962                 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
963
964                 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad,        /*  actual payload */
965                                                 &av,    /* address vector */
966                                                 wrid,   /* casting the mad wrapper pointer for err cb */
967                                                 p_vend->timeout, qpn);
968         }
969
970         if (mgt_res != IB_MGT_OK) {
971                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
972                         "osm_mtl_send_mad: ERR 3C26: "
973                         "Error sending mad (%d).\n", mgt_res);
974                 if (p_vw->p_resp_madw)
975                         osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
976                 status = IB_ERROR;
977                 goto Exit;
978         }
979
980         status = IB_SUCCESS;
981
982 Exit:
983         OSM_LOG_EXIT(p_vend->p_log);
984         return (status);
985 }
986
987 /**********************************************************************
988 Send a MAD through.
989
990 What is unclear to me is the need for the setting of all the MAD Wrapper
991 fields. Seems like the OSM uses these values during it's processing...
992 **********************************************************************/
993 ib_api_status_t
994 osm_vendor_send(IN osm_bind_handle_t h_bind,
995                 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
996 {
997         osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind;
998         osm_vendor_t *const p_vend = p_bind->p_vend;
999         osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1000         ib_api_status_t status;
1001
1002         OSM_LOG_ENTER(p_vend->p_log);
1003
1004         /*
1005          * If a response is expected to this MAD, then preallocate
1006          * a mad wrapper to contain the wire MAD received in the
1007          * response.  Allocating a wrapper here allows for easier
1008          * failure paths than after we already received the wire mad.
1009          */
1010         if (resp_expected == TRUE) {
1011                 /* we track it in the vendor wrapper */
1012                 p_vw->p_resp_madw =
1013                     osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
1014                 if (p_vw->p_resp_madw == NULL) {
1015                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
1016                                 "osm_vendor_send: ERR 3C27: "
1017                                 "Unable to allocate MAD wrapper.\n");
1018                         status = IB_INSUFFICIENT_RESOURCES;
1019                         goto Exit;
1020                 }
1021
1022                 /* put some minimal info on that wrapper */
1023                 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
1024
1025                 /* we also want to track it in the TID based map */
1026                 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t)
1027                                                          p_bind, p_madw);
1028                 if (status != IB_SUCCESS) {
1029                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
1030                                 "osm_vendor_send: ERR 3C25: "
1031                                 "Error inserting request madw by TID (%d).\n",
1032                                 status);
1033                 }
1034
1035         } else
1036                 p_vw->p_resp_madw = NULL;
1037
1038         /* do the actual send */
1039         status = osm_mtl_send_mad(p_bind, p_madw);
1040
1041 Exit:
1042         OSM_LOG_EXIT(p_vend->p_log);
1043         return (status);
1044 }
1045
1046 /**********************************************************************
1047  * the idea here is to change the content of the bind such that it
1048  * will hold the local address used for sending directed route by the SMA.
1049  **********************************************************************/
1050 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1051 {
1052         osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend;
1053
1054         OSM_LOG_ENTER(p_vend->p_log);
1055
1056         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
1057                 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
1058
1059         OSM_LOG_EXIT(p_vend->p_log);
1060
1061         return (IB_SUCCESS);
1062 }
1063
1064 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1065 {
1066         osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
1067         osm_vendor_t *p_vend = p_bind->p_vend;
1068         VAPI_ret_t status;
1069         VAPI_hca_attr_t attr_mod;
1070         VAPI_hca_attr_mask_t attr_mask;
1071
1072         OSM_LOG_ENTER(p_vend->p_log);
1073
1074         memset(&attr_mod, 0, sizeof(attr_mod));
1075         memset(&attr_mask, 0, sizeof(attr_mask));
1076
1077         attr_mod.is_sm = is_sm_val;
1078         attr_mask = HCA_ATTR_IS_SM;
1079
1080         status =
1081             VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
1082                                  &attr_mask);
1083         if (status != VAPI_OK) {
1084                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1085                         "osm_vendor_set_sm: ERR 3C28: "
1086                         "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
1087                         is_sm_val, status);
1088         }
1089
1090         OSM_LOG_EXIT(p_vend->p_log);
1091 }
1092
1093 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1094 {
1095
1096 }
1097
1098 #endif                          /* OSM_VENDOR_INTF_TEST */