contrib/ofed/management/opensm/libvendor/osm_vendor_mlx_ts_anafa.c

   1 /*
   2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
   4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  */
  35
  36 /*  AUTHOR                 Edward Bortnikov
  37  *
  38  *  DESCRIPTION
  39  *     The lower-level MAD transport interface implementation
  40  *     that allows sending a single MAD/receiving a callback
  41  *     when a single MAD is received.
  42  */
  43
  44 #if HAVE_CONFIG_H
  45 #  include <config.h>
  46 #endif                          /* HAVE_CONFIG_H */
  47
  48 #include <sys/types.h>
  49 #include <sys/stat.h>
  50 #include <sys/ioctl.h>
  51 #include <fcntl.h>
  52 #include <stddef.h>
  53 #include <errno.h>
  54 #include <stdlib.h>
  55 #include <string.h>
  56
  57 #include <vendor/osm_vendor_api.h>
  58 #include <vendor/osm_vendor_mlx_transport.h>
  59 #include <vendor/osm_vendor_mlx_transport_anafa.h>
  60 #include <vendor/osm_vendor_mlx_dispatcher.h>
  61 #include <vendor/osm_vendor_mlx_svc.h>
  62 #include <vendor/osm_ts_useraccess.h>
  63
  64 static void
  65 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
  66                                           IN struct ib_mad *p_mad,
  67                                           IN uint8_t is_smi,
  68                                           OUT osm_mad_addr_t * p_mad_addr);
  69
  70 static void
  71 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t *
  72                                           p_mad_addr, IN uint8_t is_smi,
  73                                           OUT struct ib_mad *p_mad);
  74
  75 void __osmv_TOPSPIN_ANAFA_receiver_thr(void *p_ctx)
  76 {
  77         int ts_ret_code;
  78         struct ib_mad mad;
  79         osm_mad_addr_t mad_addr;
  80         osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
  81         ib_api_status_t status = IB_SUCCESS;
  82
  83         OSM_LOG_ENTER(p_bo->p_vendor->p_log);
  84
  85         /* Make sure the p_bo object is still relevant */
  86         if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
  87                 return;
  88
  89         /* we set the type of cancelation for this thread */
  90         /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
  91
  92         while (1) {
  93                 /* Make sure the p_bo object is still relevant */
  94                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
  95                         return;
  96
  97                 /* we read one mad at a time and pass it to the read callback function */
  98                 ts_ret_code =
  99                     read(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
 100                                                                   p_transp_mgr))->
 101                          device_fd, &mad, sizeof(mad));
 102
 103                 /* Make sure the p_bo object is still relevant */
 104                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 105                         return;
 106
 107                 if (ts_ret_code != sizeof(mad)) {
 108                         osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 109                                 "__osmv_TOPSPIN_ANAFA_receiver_thr: ERR 6903: "
 110                                 "error with read, bytes = %d\n", ts_ret_code);
 111                         break;
 112                 } else {
 113                         osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 114                                 "__osmv_TOPSPIN_ANAFA_receiver_thr: "
 115                                 "MAD QPN:%d SLID:0x%04x class:0x%02x "
 116                                 "method:0x%02x attr:0x%04x status:0x%04x "
 117                                 "tid:0x%016" PRIx64 "\n",
 118                                 mad.dqpn,
 119                                 cl_ntoh16(mad.slid),
 120                                 mad.mgmt_class,
 121                                 mad.r_method,
 122                                 cl_ntoh16(mad.attribute_id),
 123                                 cl_ntoh16(mad.status),
 124                                 cl_ntoh64(mad.transaction_id));
 125
 126                         /* first arrange an address */
 127                         __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr
 128                             (p_bo->p_vendor, &mad,
 129                              (((ib_mad_t *) & mad)->mgmt_class ==
 130                               IB_MCLASS_SUBN_LID)
 131                              || (((ib_mad_t *) & mad)->mgmt_class ==
 132                                  IB_MCLASS_SUBN_DIR), &mad_addr);
 133
 134                         /* call the receiver callback */
 135
 136                         status =
 137                             osmv_dispatch_mad((osm_bind_handle_t) p_bo,
 138                                               (void *)&mad, &mad_addr);
 139
 140                         /* Make sure the p_bo object is still relevant */
 141                         if (p_bo->magic_ptr != p_bo)
 142                                 return;
 143
 144                         if (IB_INTERRUPTED == status) {
 145
 146                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 147                                         "__osmv_TOPSPIN_ANAFA_receiver_thr: "
 148                                         "The bind handle %p is being closed. "
 149                                         "Breaking the loop.\n", p_bo);
 150                                 break;
 151                         }
 152                 }
 153         }
 154
 155         OSM_LOG_EXIT(p_bo->p_vendor->p_log);
 156 }
 157
 158 /*
 159  * NAME
 160  *   osmv_transport_init
 161  *
 162  * DESCRIPTION
 163  *   Setup the MAD transport infrastructure (filters, callbacks etc).
 164  */
 165
 166 ib_api_status_t
 167 osmv_transport_init(IN osm_bind_info_t * p_info,
 168                     IN char hca_id[VENDOR_HCA_MAXNAMES],
 169                     IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
 170 {
 171         cl_status_t cl_st;
 172
 173         int ts_ioctl_ret;
 174         int device_fd;
 175         char *device_file = "/dev/ts_ua0";
 176         osm_ts_user_mad_filter filter;
 177         osmv_TOPSPIN_ANAFA_transport_mgr_t *p_mgr;
 178         osmv_TOPSPIN_ANAFA_transport_info_t *p_tpot_info;
 179         p_tpot_info =
 180             (osmv_TOPSPIN_ANAFA_transport_info_t *) p_bo->p_vendor->
 181             p_transport_info;
 182
 183         p_mgr = malloc(sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
 184         if (!p_mgr) {
 185                 return IB_INSUFFICIENT_MEMORY;
 186         }
 187
 188         memset(p_mgr, 0, sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
 189
 190         /* open TopSpin file device */
 191         device_fd = open(device_file, O_RDWR);
 192         if (device_fd < 0) {
 193                 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
 194                         device_file, errno);
 195                 return IB_ERROR;
 196         }
 197         p_mgr->device_fd = device_fd;
 198
 199         /*
 200          * Create the MAD filter on this file handle.
 201          */
 202
 203         filter.port = 0;        /* Victor */
 204         filter.direction = TS_IB_MAD_DIRECTION_IN;
 205         filter.mask =
 206             TS_IB_MAD_FILTER_DIRECTION |
 207             TS_IB_MAD_FILTER_PORT |
 208             TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
 209
 210         switch (p_info->mad_class) {
 211         case IB_MCLASS_SUBN_LID:
 212         case IB_MCLASS_SUBN_DIR:
 213                 filter.qpn = 0;
 214                 filter.mgmt_class = IB_MCLASS_SUBN_LID;
 215                 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
 216                 if (ts_ioctl_ret < 0) {
 217                         return IB_ERROR;
 218                 }
 219
 220                 filter.mgmt_class = IB_MCLASS_SUBN_DIR;
 221                 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
 222                 if (ts_ioctl_ret < 0) {
 223                         return IB_ERROR;
 224                 }
 225
 226                 break;
 227
 228         case IB_MCLASS_SUBN_ADM:
 229         default:
 230                 filter.qpn = 1;
 231                 filter.mgmt_class = p_info->mad_class;
 232                 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
 233                 if (ts_ioctl_ret < 0) {
 234                         return IB_ERROR;
 235                 }
 236                 break;
 237         }
 238
 239         p_bo->p_transp_mgr = p_mgr;
 240
 241         /* Initialize the magic_ptr to the pointer of the p_bo info.
 242            This will be used to signal when the object is being destroyed, so no
 243            real action will be done then. */
 244         p_bo->magic_ptr = p_bo;
 245
 246         /* init receiver thread */
 247         cl_st =
 248             cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_ANAFA_receiver_thr,
 249                            (void *)p_bo, "osmv TOPSPIN_ANAFA rcv thr");
 250
 251         return (ib_api_status_t) cl_st;
 252 }
 253
 254 /*
 255  * NAME
 256  *   osmv_transport_send_mad
 257  *
 258  * DESCRIPTION
 259  *   Send a single MAD (256 byte)
 260  */
 261
 262 ib_api_status_t
 263 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
 264                         IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
 265 {
 266
 267         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 268         osm_vendor_t const *p_vend = p_bo->p_vendor;
 269         struct ib_mad ts_mad = { 0 };
 270         int ret;
 271         ib_api_status_t status;
 272
 273         const ib_mad_t *p_mad_hdr = p_mad;
 274
 275         OSM_LOG_ENTER(p_vend->p_log);
 276
 277         /* Make sure the p_bo object is still relevant */
 278         if (p_bo->magic_ptr != p_bo)
 279                 return IB_INVALID_CALLBACK;
 280
 281         /*
 282          * Copy the MAD over to the sent mad
 283          */
 284         memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
 285
 286         /*
 287          * For all sends other than directed route SM MADs,
 288          * acquire an address vector for the destination.
 289          */
 290         if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
 291
 292                 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(p_mad_addr,
 293                                                           p_mad_hdr->
 294                                                           mgmt_class ==
 295                                                           IB_MCLASS_SUBN_LID,
 296                                                           &ts_mad);
 297         } else {
 298                 /* is a directed route - we need to construct a permissive address */
 299                 /* we do not need port number since it is part of the mad_hndl */
 300                 ts_mad.dlid = IB_LID_PERMISSIVE;
 301                 ts_mad.slid = IB_LID_PERMISSIVE;
 302         }
 303         if ((p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_DIR) ||
 304             (p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_LID)) {
 305                 ts_mad.sqpn = 0;
 306                 ts_mad.dqpn = 0;
 307         } else {
 308                 ts_mad.sqpn = 1;
 309                 ts_mad.dqpn = 1;
 310         }
 311
 312         /* ts_mad.port = p_bo->port_num; */
 313         ts_mad.port = 0;        /* Victor */
 314
 315         /* send it */
 316         ret =
 317             write(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
 318                                                            p_transp_mgr))->
 319                   device_fd, &ts_mad, sizeof(ts_mad));
 320
 321         if (ret != sizeof(ts_mad)) {
 322                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
 323                         "osmv_transport_mad_send: ERR 6904: "
 324                         "Error sending mad (%d).\n", ret);
 325                 status = IB_ERROR;
 326                 goto Exit;
 327         }
 328
 329         status = IB_SUCCESS;
 330
 331 Exit:
 332         OSM_LOG_EXIT(p_vend->p_log);
 333         return (status);
 334 }
 335
 336 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
 337 {
 338         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 339         osmv_TOPSPIN_ANAFA_transport_mgr_t *p_tpot_mgr =
 340             (osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->p_transp_mgr);
 341
 342         CL_ASSERT(p_bo);
 343
 344         /* First of all - zero out the magic_ptr, so if a callback is called -
 345            it'll know that we are currently closing down, and will not handle the
 346            mad. */
 347         p_bo->magic_ptr = 0;
 348
 349         /* usleep(3000000); */
 350
 351         /* pthread_cancel (p_tpot_mgr->receiver.osd.id); */
 352         cl_thread_destroy(&(p_tpot_mgr->receiver));
 353         free(p_tpot_mgr);
 354 }
 355
 356 static void
 357 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
 358                                           IN uint8_t is_smi,
 359                                           OUT struct ib_mad *p_mad)
 360 {
 361
 362         /* For global destination or Multicast address: */
 363         p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
 364         p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
 365         if (is_smi) {
 366                 p_mad->sqpn = 0;
 367                 p_mad->dqpn = 0;
 368         } else {
 369                 p_mad->sqpn = 1;
 370                 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp;
 371         }
 372         /*
 373            HACK we limit to the first PKey Index assuming it will
 374            always be the default PKey
 375          */
 376         p_mad->pkey_index = 0;
 377 }
 378
 379 static void
 380 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
 381                                           IN struct ib_mad *p_mad,
 382                                           IN uint8_t is_smi,
 383                                           OUT osm_mad_addr_t * p_mad_addr)
 384 {
 385         p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
 386         p_mad_addr->static_rate = 0;
 387         p_mad_addr->path_bits = 0;
 388         if (is_smi) {
 389                 /* SMI */
 390                 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
 391                 p_mad_addr->addr_type.smi.port_num = p_mad->port;
 392         } else {
 393                 /* GSI */
 394                 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn;
 395                 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
 396                 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
 397                 p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
 398
 399                 p_mad_addr->addr_type.gsi.global_route = FALSE;
 400                 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
 401                 /*
 402                    if (p_mad_addr->addr_type.gsi.global_route)
 403                    {
 404                    p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
 405                    ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
 406                    p_rcv_desc->grh.traffic_class,
 407                    p_rcv_desc->grh.flow_label);
 408                    p_mad_addr->addr_type.gsi.grh_info.hop_limit =  p_rcv_desc->grh.hop_limit;
 409                    memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
 410                    &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
 411                    memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
 412                    p_rcv_desc->grh.dgid,  sizeof(ib_net64_t));
 413                    }
 414                  */
 415         }
 416 }