contrib/ofed/management/opensm/libvendor/osm_vendor_mlx_ibmgt.c

   1 /*
   2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
   4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  */
  35
  36 /*  AUTHOR                 Edward Bortnikov
  37  *
  38  *  DESCRIPTION
  39  *     The lower-level MAD transport interface implementation
  40  *     that allows sending a single MAD/receiving a callback
  41  *     when a single MAD is received.
  42  */
  43
  44 #if HAVE_CONFIG_H
  45 #  include <config.h>
  46 #endif                          /* HAVE_CONFIG_H */
  47
  48 #include <stdlib.h>
  49 #include <string.h>
  50 #include <ib_mgt.h>
  51 #include <complib/cl_event.h>
  52 #include <vendor/osm_vendor_mlx_transport.h>
  53 #include <vendor/osm_vendor_mlx_dispatcher.h>
  54 #include <opensm/osm_log.h>
  55
  56 typedef struct _osmv_IBMGT_transport_mgr_ {
  57         IB_MGT_mad_type_t mad_type;
  58         uint8_t mgmt_class;     /* for gsi */
  59         /* for communication between send call back and send mad */
  60         boolean_t is_send_ok;
  61         cl_event_t send_done;
  62 } osmv_IBMGT_transport_mgr_t;
  63
  64 typedef struct _osmv_IBMGT_transport_info_ {
  65         IB_MGT_mad_hndl_t smi_h;
  66         cl_qlist_t *p_smi_list;
  67
  68         IB_MGT_mad_hndl_t gsi_h;
  69         /* holds bind object list for every binded mgmt class */
  70         cl_qlist_t *gsi_mgmt_lists[15];
  71 } osmv_IBMGT_transport_info_t;
  72
  73 static void
  74 __osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
  75                                   IN uint8_t is_smi,
  76                                   OUT osm_mad_addr_t * p_mad_addr);
  77
  78 static void
  79 __osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr,
  80                                     IN uint8_t is_smi, OUT IB_ud_av_t * p_av);
  81
  82 void
  83 __osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl,
  84                      IN u_int64_t wrid,
  85                      IN IB_comp_status_t status, IN void *private_ctx_p);
  86
  87 void
  88 __osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl,
  89                     IN void *private_ctx_p,
  90                     IN void *payload_p,
  91                     IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p);
  92
  93 /*
  94  * NAME
  95  *   osmv_transport_init
  96  *
  97  * DESCRIPTION
  98  *   Setup the MAD transport infrastructure (filters, callbacks etc).
  99  */
 100
 101 ib_api_status_t
 102 osmv_transport_init(IN osm_bind_info_t * p_info,
 103                     IN char hca_id[VENDOR_HCA_MAXNAMES],
 104                     IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
 105 {
 106         ib_api_status_t st = IB_SUCCESS;
 107         IB_MGT_ret_t ret;
 108         IB_MGT_mad_type_t mad_type;
 109         osmv_IBMGT_transport_mgr_t *p_mgr;
 110         osmv_IBMGT_transport_info_t *p_tpot_info;
 111         cl_list_obj_t *p_obj = NULL;
 112         osm_log_t *p_log = p_bo->p_vendor->p_log;
 113         int i;
 114
 115         UNUSED_PARAM(hca_idx);
 116
 117         /* if first bind, allocate tranport_info at vendor */
 118         if (NULL == p_bo->p_vendor->p_transport_info) {
 119                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 120                         "osmv_transport_init: first bind() for the vendor\n");
 121                 p_bo->p_vendor->p_transport_info
 122                     = (osmv_IBMGT_transport_info_t *)
 123                     malloc(sizeof(osmv_IBMGT_transport_info_t));
 124                 if (NULL == p_bo->p_vendor->p_transport_info) {
 125                         return IB_INSUFFICIENT_MEMORY;
 126                 }
 127                 memset(p_bo->p_vendor->p_transport_info, 0,
 128                        sizeof(osmv_IBMGT_transport_info_t));
 129                 p_tpot_info =
 130                     (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->
 131                                                      p_transport_info);
 132
 133                 p_tpot_info->smi_h = 0xffffffff;
 134                 p_tpot_info->p_smi_list = NULL;
 135
 136                 p_tpot_info->gsi_h = 0xffffffff;
 137                 for (i = 0; i < 15; i++) {
 138
 139                         p_tpot_info->gsi_mgmt_lists[i] = NULL;
 140                 }
 141
 142         } else {
 143
 144                 p_tpot_info =
 145                     (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->
 146                                                      p_transport_info);
 147         }
 148
 149         /* Initialize the magic_ptr to the pointer of the p_bo info.
 150            This will be used to signal when the object is being destroyed, so no
 151            real action will be done then. */
 152         p_bo->magic_ptr = p_bo;
 153
 154         /* allocate transport mgr */
 155         p_mgr = malloc(sizeof(osmv_IBMGT_transport_mgr_t));
 156         if (NULL == p_mgr) {
 157                 free(p_tpot_info);
 158                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 159                         "osmv_transport_init: ERR 7201: " "alloc failed \n");
 160                 return IB_INSUFFICIENT_MEMORY;
 161         }
 162
 163         memset(p_mgr, 0, sizeof(osmv_IBMGT_transport_mgr_t));
 164
 165         p_bo->p_transp_mgr = p_mgr;
 166
 167         switch (p_info->mad_class) {
 168         case IB_MCLASS_SUBN_LID:
 169         case IB_MCLASS_SUBN_DIR:
 170                 mad_type = IB_MGT_SMI;
 171                 break;
 172
 173         case IB_MCLASS_SUBN_ADM:
 174         default:
 175                 mad_type = IB_MGT_GSI;
 176                 break;
 177         }
 178
 179         /* we only support one class registration per SMI/GSI !!! */
 180         switch (mad_type) {
 181         case IB_MGT_SMI:
 182                 /* we do not need to bind the handle if already available */
 183                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 184                         "osmv_transport_init: SMI bind\n");
 185
 186                 if (p_tpot_info->smi_h == 0xffffffff) {
 187                         ret = IB_MGT_get_handle(hca_id,
 188                                                 p_bo->port_num,
 189                                                 IB_MGT_SMI,
 190                                                 &(p_tpot_info->smi_h));
 191                         if (IB_MGT_OK != ret) {
 192                                 osm_log(p_log, OSM_LOG_ERROR,
 193                                         "osmv_transport_init: ERR 7202: "
 194                                         "IB_MGT_get_handle for smi failed \n");
 195                                 st = IB_ERROR;
 196                                 free(p_mgr);
 197                                 goto Exit;
 198                         }
 199
 200                         osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 201                                 "osmv_transport_init: got smi handle:%d \n",
 202                                 p_tpot_info->smi_h);
 203
 204                         ret = IB_MGT_bind_sm(p_tpot_info->smi_h);
 205                         if (IB_MGT_OK != ret) {
 206                                 osm_log(p_log, OSM_LOG_ERROR,
 207                                         "osmv_transport_init: ERR 7203: "
 208                                         "IB_MGT_bind_sm failed \n");
 209                                 st = IB_ERROR;
 210                                 free(p_mgr);
 211                                 goto Exit;
 212                         }
 213
 214                         /* init smi list */
 215                         p_tpot_info->p_smi_list = malloc(sizeof(cl_qlist_t));
 216                         if (NULL == p_tpot_info->p_smi_list) {
 217                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 218                                         "osmv_transport_init: ERR 7204: "
 219                                         "alloc failed \n");
 220                                 IB_MGT_unbind_sm(p_tpot_info->smi_h);
 221                                 IB_MGT_release_handle(p_tpot_info->smi_h);
 222                                 free(p_mgr);
 223                                 return IB_INSUFFICIENT_MEMORY;
 224                         }
 225                         memset(p_tpot_info->p_smi_list, 0, sizeof(cl_qlist_t));
 226                         cl_qlist_init(p_tpot_info->p_smi_list);
 227
 228                         osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 229                                 "osmv_transport_init: before reg_cb\n");
 230                         ret = IB_MGT_reg_cb(p_tpot_info->smi_h,
 231                                             &__osmv_IBMGT_rcv_cb,
 232                                             p_bo,
 233                                             &__osmv_IBMGT_send_cb,
 234                                             p_tpot_info->p_smi_list,
 235                                             IB_MGT_RCV_CB_MASK |
 236                                             IB_MGT_SEND_CB_MASK);
 237                         if (ret != IB_SUCCESS) {
 238                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 239                                         "osmv_transport_init: ERR 7205: "
 240                                         "reg_cb failed with return code:%x \n",
 241                                         ret);
 242                                 IB_MGT_unbind_sm(p_tpot_info->smi_h);
 243                                 IB_MGT_release_handle(p_tpot_info->smi_h);
 244                                 free(p_tpot_info->p_smi_list);
 245                                 free(p_mgr);
 246                                 st = IB_ERROR;
 247                                 goto Exit;
 248                         }
 249
 250                 }
 251                 /* insert to list of smi's - for raising callbacks later on */
 252                 p_obj = malloc(sizeof(cl_list_obj_t));
 253                 if (p_obj)
 254                         memset(p_obj, 0, sizeof(cl_list_obj_t));
 255                 cl_qlist_set_obj(p_obj, p_bo);
 256                 cl_qlist_insert_tail(p_tpot_info->p_smi_list,
 257                                      &p_obj->list_item);
 258
 259                 break;
 260
 261         case IB_MGT_GSI:
 262                 /* we do not need to bind the handle if already available */
 263                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 264                         "osmv_transport_init: ERR 7206: GSI bind\n");
 265                 if (p_tpot_info->gsi_h == 0xffffffff) {
 266                         ret = IB_MGT_get_handle(hca_id,
 267                                                 p_bo->port_num,
 268                                                 IB_MGT_GSI,
 269                                                 &(p_tpot_info->gsi_h));
 270                         if (IB_MGT_OK != ret) {
 271                                 osm_log(p_log, OSM_LOG_ERROR,
 272                                         "osmv_transport_init: ERR 7207: "
 273                                         "IB_MGT_get_handle for gsi failed \n");
 274                                 st = IB_ERROR;
 275                                 free(p_mgr);
 276                                 goto Exit;
 277                         }
 278                 }
 279
 280                 /* this mgmt class was not binded yet */
 281                 if (p_tpot_info->gsi_mgmt_lists[p_info->mad_class] == NULL) {
 282                         ret =
 283                             IB_MGT_bind_gsi_class(p_tpot_info->gsi_h,
 284                                                   p_info->mad_class);
 285                         if (IB_MGT_OK != ret) {
 286                                 osm_log(p_log, OSM_LOG_ERROR,
 287                                         "osmv_transport_init: ERR 7208: "
 288                                         "IB_MGT_bind_gsi_class failed \n");
 289                                 st = IB_ERROR;
 290                                 free(p_mgr);
 291                                 goto Exit;
 292                         }
 293
 294                         p_tpot_info->gsi_mgmt_lists[p_info->mad_class] =
 295                             malloc(sizeof(cl_qlist_t));
 296                         if (NULL ==
 297                             p_tpot_info->gsi_mgmt_lists[p_info->mad_class]) {
 298                                 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h,
 299                                                         p_info->mad_class);
 300                                 free(p_mgr);
 301                                 return IB_INSUFFICIENT_MEMORY;
 302                         }
 303                         memset(p_tpot_info->gsi_mgmt_lists[p_info->mad_class],
 304                                0, sizeof(cl_qlist_t));
 305                         cl_qlist_init(p_tpot_info->
 306                                       gsi_mgmt_lists[p_info->mad_class]);
 307                 }
 308                 /* insert to list of smi's - for raising callbacks later on */
 309                 p_obj = malloc(sizeof(cl_list_obj_t));
 310                 if (p_obj)
 311                         memset(p_obj, 0, sizeof(cl_list_obj_t));
 312                 cl_qlist_set_obj(p_obj, p_bo);
 313                 cl_qlist_insert_tail(p_tpot_info->
 314                                      gsi_mgmt_lists[p_info->mad_class],
 315                                      &p_obj->list_item);
 316
 317                 p_mgr->mgmt_class = p_info->mad_class;
 318                 ret = IB_MGT_reg_cb(p_tpot_info->gsi_h,
 319                                     &__osmv_IBMGT_rcv_cb,
 320                                     p_bo,
 321                                     &__osmv_IBMGT_send_cb,
 322                                     p_bo,
 323                                     IB_MGT_RCV_CB_MASK | IB_MGT_SEND_CB_MASK);
 324
 325                 if (ret != IB_SUCCESS) {
 326                         IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h,
 327                                                 p_mgr->mgmt_class);
 328                         free(p_tpot_info->gsi_mgmt_lists[p_mgr->mgmt_class]);
 329                         free(p_mgr);
 330                         st = IB_ERROR;
 331                         goto Exit;
 332                 }
 333
 334                 break;
 335
 336         default:
 337                 osm_log(p_log, OSM_LOG_ERROR,
 338                         "osmv_transport_init: ERR 7209: unrecognized mgmt class \n");
 339                 st = IB_ERROR;
 340                 free(p_mgr);
 341                 goto Exit;
 342         }
 343
 344         osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 345                 "osmv_transport_init: GSI bind\n");
 346         cl_event_construct(&p_mgr->send_done);
 347         cl_event_init(&p_mgr->send_done, TRUE);
 348         p_mgr->is_send_ok = FALSE;
 349         p_mgr->mad_type = mad_type;
 350
 351 Exit:
 352         /* OSM_LOG_EXIT(p_log ); */
 353         return (ib_api_status_t) st;
 354 }
 355
 356 /*
 357  * NAME
 358  *   osmv_transport_send_mad
 359  *
 360  * DESCRIPTION
 361  *   Send a single MAD (256 byte)
 362  */
 363
 364 ib_api_status_t
 365 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
 366                         IN void *p_ib_mad, IN const osm_mad_addr_t * p_mad_addr)
 367 {
 368
 369         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 370         osmv_IBMGT_transport_info_t *p_tpot_info =
 371             (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->p_transport_info);
 372         osm_vendor_t const *p_vend = p_bo->p_vendor;
 373         ib_api_status_t status;
 374         IB_ud_av_t av;
 375         IB_MGT_ret_t ret;
 376         ib_mad_t *p_mad = p_ib_mad;
 377
 378         OSM_LOG_ENTER(p_vend->p_log);
 379
 380         CL_ASSERT(p_bo->p_vendor->p_transport_info);
 381
 382         /*
 383          * For all sends other than directed route SM MADs,
 384          * acquire an address vector for the destination.
 385          */
 386         if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
 387                 __osmv_IBMGT_osm_addr_to_ibmgt_addr(p_mad_addr,
 388                                                     p_mad->mgmt_class ==
 389                                                     IB_MCLASS_SUBN_LID, &av);
 390         } else {
 391                 /* is a directed route - we need to construct a permissive address */
 392                 memset(&av, 0, sizeof(av));
 393                 /* we do not need port number since it is part of the mad_hndl */
 394                 av.dlid = IB_LID_PERMISSIVE;
 395         }
 396
 397         /* send it */
 398         if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
 399             (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
 400
 401                 /* SMI CASE */
 402                 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
 403                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
 404                                 "osmv_transport_mad_send: "
 405                                 "av.dlid:0x%X, "
 406                                 "av.static_rate:%d, "
 407                                 "av.path_bits:%d.\n",
 408                                 cl_ntoh16(av.dlid), av.static_rate,
 409                                 av.src_path_bits);
 410                 }
 411
 412                 ret = IB_MGT_send_mad(p_tpot_info->smi_h, p_mad,        /*  actual payload */
 413                                       &av,      /*  address vector */
 414                                       (u_int64_t) CAST_P2LONG(p_bo),
 415                                       IB_MGT_DEFAULT_SEND_TIME);
 416         } else {
 417                 /* GSI CASE - Support Remote QP */
 418                 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
 419                         osm_log(p_vend->p_log, OSM_LOG_DEBUG,
 420                                 "osmv_transport_mad_send: "
 421                                 "av.dlid:0x%X, av.static_rate:%d, av.path_bits:%d, remote qp:%d \n",
 422                                 cl_ntoh16(av.dlid), av.static_rate,
 423                                 av.src_path_bits,
 424                                 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
 425                             );
 426                 }
 427
 428                 ret = IB_MGT_send_mad_to_qp(p_tpot_info->gsi_h, p_mad,  /*  actual payload */
 429                                             &av,        /*  address vector */
 430                                             (u_int64_t) CAST_P2LONG(p_bo),
 431                                             IB_MGT_DEFAULT_SEND_TIME,
 432                                             cl_ntoh32(p_mad_addr->addr_type.gsi.
 433                                                       remote_qp));
 434
 435         }
 436
 437         status = IB_SUCCESS;
 438         if (ret != IB_MGT_OK) {
 439                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
 440                         "osmv_transport_mad_send: ERR 7210: "
 441                         "Error sending mad (%d).\n", ret);
 442                 status = IB_ERROR;
 443         } else {
 444                 osmv_IBMGT_transport_mgr_t *p_mgr =
 445                     (osmv_IBMGT_transport_mgr_t *) (p_bo->p_transp_mgr);
 446
 447                 /* Let the others work when I am sleeping ... */
 448                 osmv_txn_unlock(p_bo);
 449
 450                 cl_event_wait_on(&(p_mgr->send_done), 0xffffffff, TRUE);
 451
 452                 /* Re-acquire the lock */
 453                 osmv_txn_lock(p_bo);
 454
 455                 if (TRUE == p_bo->is_closing) {
 456
 457                         osm_log(p_vend->p_log, OSM_LOG_ERROR,
 458                                 "osmv_transport_mad_send: ERR 7211: "
 459                                 "The handle %p is being unbound, cannot send.\n",
 460                                 h_bind);
 461                         status = IB_ERROR;
 462                 }
 463
 464                 if (p_mgr->is_send_ok == FALSE) {
 465                         status = IB_ERROR;
 466                 }
 467         }
 468
 469         OSM_LOG_EXIT(p_vend->p_log);
 470         return (status);
 471 }
 472
 473 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
 474 {
 475         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 476         osm_log_t *p_log = p_bo->p_vendor->p_log;
 477         osmv_IBMGT_transport_mgr_t *p_mgr;
 478         osmv_IBMGT_transport_info_t *p_tpot_info;
 479         IB_MGT_ret_t ret;
 480         cl_list_obj_t *p_obj = NULL;
 481         cl_list_item_t *p_item, *p_item_tmp;
 482         int i;
 483         cl_qlist_t *p_list = NULL;
 484
 485         OSM_LOG_ENTER(p_log);
 486
 487         CL_ASSERT(p_bo);
 488
 489         /* First of all - zero out the magic_ptr, so if a callback is called -
 490            it'll know that we are currently closing down, and will not handle the
 491            mad. */
 492         p_bo->magic_ptr = 0;
 493
 494         p_mgr = (osmv_IBMGT_transport_mgr_t *) (p_bo->p_transp_mgr);
 495         p_tpot_info =
 496             (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->p_transport_info);
 497
 498         switch (p_mgr->mad_type) {
 499         case IB_MGT_SMI:
 500                 p_list = p_tpot_info->p_smi_list;
 501
 502                 /* remove from the bindings list */
 503                 p_item = cl_qlist_head(p_list);
 504                 while (p_item != cl_qlist_end(p_list)) {
 505                         p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item);
 506                         if (cl_qlist_obj(p_obj) == h_bind) {
 507                                 break;
 508                         }
 509                         p_item_tmp = cl_qlist_next(p_item);
 510                         p_item = p_item_tmp;
 511                 }
 512
 513                 CL_ASSERT(p_item != cl_qlist_end(p_list));
 514                 cl_qlist_remove_item(p_list, p_item);
 515                 if (p_obj)
 516                         free(p_obj);
 517
 518                 /* no one is binded to smi anymore - we can free the list, unbind & realease the hndl */
 519                 if (cl_is_qlist_empty(p_list) == TRUE) {
 520                         free(p_list);
 521                         p_list = NULL;
 522
 523                         ret = IB_MGT_unbind_sm(p_tpot_info->smi_h);
 524                         if (ret != IB_MGT_OK) {
 525                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 526                                         "osmv_transport_done: ERR 7212: "
 527                                         "Failed to unbind sm\n");
 528                         }
 529
 530                         ret = IB_MGT_release_handle(p_tpot_info->smi_h);
 531                         if (ret != IB_MGT_OK) {
 532                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 533                                         "osmv_transport_done: ERR 7213: "
 534                                         "Failed to release smi handle\n");
 535                         }
 536                         p_tpot_info->smi_h = 0xffffffff;
 537                 }
 538                 break;
 539
 540         case IB_MGT_GSI:
 541                 p_list = p_tpot_info->gsi_mgmt_lists[p_mgr->mgmt_class];
 542                 /* remove from the bindings list */
 543                 p_item = cl_qlist_head(p_list);
 544                 while (p_item != cl_qlist_end(p_list)) {
 545                         p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item);
 546                         if (cl_qlist_obj(p_obj) == h_bind) {
 547                                 break;
 548                         }
 549                         p_item_tmp = cl_qlist_next(p_item);
 550                         p_item = p_item_tmp;
 551                 }
 552
 553                 CL_ASSERT(p_item != cl_qlist_end(p_list));
 554                 cl_qlist_remove_item(p_list, p_item);
 555                 if (p_obj)
 556                         free(p_obj);
 557
 558                 /* no one is binded to this class anymore - we can free the list and unbind this class */
 559                 if (cl_is_qlist_empty(p_list) == TRUE) {
 560                         free(p_list);
 561                         p_list = NULL;
 562
 563                         ret =
 564                             IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h,
 565                                                     p_mgr->mgmt_class);
 566                         if (ret != IB_MGT_OK) {
 567                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 568                                         "osmv_transport_done: ERR 7214: "
 569                                         "Failed to unbind gsi class\n");
 570                         }
 571                 }
 572
 573                 /* all the mgmt classes are unbinded - release gsi handle */
 574                 for (i = 0; i < 15; i++) {
 575                         if (p_tpot_info->gsi_mgmt_lists[i] != NULL) {
 576                                 break;
 577                         }
 578                 }
 579
 580                 if (i == 15) {
 581                         ret = IB_MGT_release_handle(p_tpot_info->gsi_h);
 582                         if (ret != IB_MGT_OK) {
 583                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 584                                         "osmv_transport_done: ERR 7215: "
 585                                         "Failed to release gsi handle\n");
 586                         }
 587                         p_tpot_info->gsi_h = 0xffffffff;
 588                 }
 589         }                       /* end switch */
 590
 591         free(p_mgr);
 592 }
 593
 594 /**********************************************************************
 595  * IB_MGT Receive callback : invoked after each receive
 596  **********************************************************************/
 597 void
 598 __osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl,
 599                     IN void *private_ctx_p,
 600                     IN void *payload_p,
 601                     IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
 602 {
 603         osmv_bind_obj_t *p_bo;
 604         osm_mad_addr_t mad_addr;
 605         cl_list_item_t *p_item;
 606         cl_list_obj_t *p_obj;
 607         cl_qlist_t *p_list;
 608         ib_mad_t *p_mad = (ib_mad_t *) payload_p;
 609         osm_vendor_t *p_vendor;
 610         osmv_IBMGT_transport_info_t *p_tinfo;
 611
 612         __osmv_IBMGT_rcv_desc_to_osm_addr(rcv_remote_info_p,
 613                                           ((p_mad->mgmt_class ==
 614                                             IB_MCLASS_SUBN_LID)
 615                                            || (p_mad->mgmt_class ==
 616                                                IB_MCLASS_SUBN_DIR)), &mad_addr);
 617
 618         /* different handling of SMI and GSI */
 619         if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
 620             (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
 621                 /* SMI CASE */
 622                 p_bo = (osmv_bind_obj_t *) private_ctx_p;
 623                 /* Make sure the p_bo object is still relevant */
 624                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 625                         return;
 626
 627                 p_vendor = p_bo->p_vendor;
 628                 p_tinfo =
 629                     (osmv_IBMGT_transport_info_t *) p_vendor->p_transport_info;
 630                 p_list = p_tinfo->p_smi_list;
 631         } else {
 632                 /* GSI CASE */
 633                 p_bo = (osmv_bind_obj_t *) private_ctx_p;
 634                 /* Make sure the p_bo object is still relevant */
 635                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 636                         return;
 637
 638                 p_vendor = p_bo->p_vendor;
 639                 p_tinfo =
 640                     (osmv_IBMGT_transport_info_t *) p_vendor->p_transport_info;
 641                 p_list = p_tinfo->gsi_mgmt_lists[p_mad->mgmt_class];
 642         }
 643
 644         /* go over the bindings list and send the mad, one of them will accept it,
 645            the others will drope
 646          */
 647         p_item = cl_qlist_head(p_list);
 648         while (p_item != cl_qlist_end(p_list)) {
 649                 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item);
 650                 p_bo = cl_qlist_obj(p_obj);
 651                 /* give upper layer the mad */
 652                 osmv_dispatch_mad((osm_bind_handle_t) p_bo, payload_p,
 653                                   &mad_addr);
 654                 /* Make sure the p_bo object is still relevant */
 655                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 656                         return;
 657
 658                 p_item = cl_qlist_next(p_item);
 659         }
 660 }
 661
 662 /**********************************************************************
 663  * IB_MGT Send callback : invoked after each send
 664  **********************************************************************/
 665 void
 666 __osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl,
 667                      IN u_int64_t wrid,
 668                      IN IB_comp_status_t status, IN void *private_ctx_p)
 669 {
 670         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) CAST_P2LONG(wrid);
 671
 672         osmv_IBMGT_transport_mgr_t *p_mgr =
 673             (osmv_IBMGT_transport_mgr_t *) p_bo->p_transp_mgr;
 674
 675         /* Make sure the p_bo object is still relevant */
 676         if (p_bo->magic_ptr != p_bo)
 677                 return;
 678
 679         /* we assume that each send on a bind object is synchronized, and no paralel sends
 680            from diffrent threads with same object can be made */
 681         if (status == IB_COMP_SUCCESS) {
 682                 p_mgr->is_send_ok = TRUE;
 683         } else
 684                 p_mgr->is_send_ok = FALSE;
 685         cl_event_signal(&p_mgr->send_done);
 686
 687 }
 688
 689 /**********************************************************************
 690  * IB_MGT to OSM ADDRESS VECTOR
 691  **********************************************************************/
 692 static void
 693 __osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
 694                                   IN uint8_t is_smi,
 695                                   OUT osm_mad_addr_t * p_mad_addr)
 696 {
 697         /*  p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
 698         p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
 699         p_mad_addr->static_rate = 0;    /*  HACK - we do not  know the rate ! */
 700         p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
 701         /* Clear the grh any way to avoid unset fields */
 702         memset(&p_mad_addr->addr_type.gsi.grh_info, 0,
 703                sizeof(p_mad_addr->addr_type.gsi.grh_info));
 704
 705         if (is_smi) {
 706                 /* SMI */
 707                 p_mad_addr->addr_type.smi.source_lid =
 708                     cl_hton16(p_rcv_desc->remote_lid);
 709                 p_mad_addr->addr_type.smi.port_num = 99;        /*  HACK - if used - should fail */
 710         } else {
 711                 /* GSI */
 712                 /* seems to me there is a IBMGT bug reversing the QPN ... */
 713                 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
 714                 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
 715
 716                 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
 717                 /*  we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
 718                 /*  the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
 719                 /*  the full PKey table - than go by the index. */
 720                 /*  since this does not seem reasonable to me I simply use the default */
 721                 /*  There is a TAVOR limitation that only one P_KEY is supported per  */
 722                 /*  QP - so QP1 must use IB_DEFAULT_PKEY */
 723                 p_mad_addr->addr_type.gsi.pkey_ix = 0;
 724                 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
 725
 726                 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
 727                 /* copy the GRH data if relevant */
 728                 if (p_mad_addr->addr_type.gsi.global_route) {
 729                         p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
 730                             ib_grh_set_ver_class_flow(p_rcv_desc->grh.
 731                                                       IP_version,
 732                                                       p_rcv_desc->grh.
 733                                                       traffic_class,
 734                                                       p_rcv_desc->grh.
 735                                                       flow_label);
 736                         p_mad_addr->addr_type.gsi.grh_info.hop_limit =
 737                             p_rcv_desc->grh.hop_limit;
 738                         memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
 739                                &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
 740                         memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
 741                                p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
 742                 }
 743         }
 744 }
 745
 746 /**********************************************************************
 747  * OSM ADDR VECTOR TO IB_MGT
 748  **********************************************************************/
 749 void
 750 __osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr,
 751                                     IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
 752 {
 753
 754         /* For global destination or Multicast address: */
 755         u_int8_t ver;
 756
 757         memset(p_av, 0, sizeof(IB_ud_av_t));
 758
 759         p_av->src_path_bits = p_mad_addr->path_bits;
 760         p_av->static_rate = p_mad_addr->static_rate;
 761         p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
 762
 763         if (is_smi) {
 764                 p_av->sl = 0;   /*  Just to note we use 0 here. */
 765         } else {
 766                 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
 767                 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
 768
 769                 if (p_mad_addr->addr_type.gsi.global_route) {
 770                         ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
 771                                                   grh_info.ver_class_flow, &ver,
 772                                                   &p_av->traffic_class,
 773                                                   &p_av->flow_label);
 774                         p_av->hop_limit =
 775                             p_mad_addr->addr_type.gsi.grh_info.hop_limit;
 776                         p_av->sgid_index = 0;   /*  we always use source GID 0 */
 777                         memcpy(&p_av->dgid,
 778                                &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
 779                                sizeof(ib_net64_t));
 780
 781                 }
 782         }
 783 }