contrib/ofed/management/opensm/libvendor/osm_vendor_mlx_ts.c

   1 /*
   2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
   4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  */
  35
  36 /*  AUTHOR                 Edward Bortnikov
  37  *
  38  *  DESCRIPTION
  39  *     The lower-level MAD transport interface implementation
  40  *     that allows sending a single MAD/receiving a callback
  41  *     when a single MAD is received.
  42  */
  43
  44 #if HAVE_CONFIG_H
  45 #  include <config.h>
  46 #endif                          /* HAVE_CONFIG_H */
  47
  48 #include <sys/types.h>
  49 #include <sys/stat.h>
  50 #include <sys/ioctl.h>
  51 #include <fcntl.h>
  52 #include <errno.h>
  53 #include <stdlib.h>
  54 #include <string.h>
  55
  56 #include <vendor/osm_vendor_api.h>
  57 #include <vendor/osm_vendor_mlx_transport.h>
  58 #include <vendor/osm_vendor_mlx_dispatcher.h>
  59 #include <vendor/osm_vendor_mlx_svc.h>
  60 #include <vendor/osm_ts_useraccess.h>
  61
  62 typedef struct _osmv_TOPSPIN_transport_mgr_ {
  63         int device_fd;
  64         osm_ts_user_mad_filter filter;
  65         cl_thread_t receiver;
  66 } osmv_TOPSPIN_transport_mgr_t;
  67
  68 static void
  69 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
  70                                     IN struct ib_mad *p_mad,
  71                                     IN uint8_t is_smi,
  72                                     OUT osm_mad_addr_t * p_mad_addr);
  73
  74 static void
  75 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
  76                                     IN uint8_t is_smi,
  77                                     OUT struct ib_mad *p_mad);
  78
  79 void __osmv_TOPSPIN_receiver_thr(void *p_ctx)
  80 {
  81         int ts_ret_code;
  82         struct ib_mad mad;
  83         osm_mad_addr_t mad_addr;
  84         osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
  85         ib_api_status_t status = IB_SUCCESS;
  86
  87         OSM_LOG_ENTER(p_bo->p_vendor->p_log);
  88
  89         /* Make sure the p_bo object is still relevant */
  90         if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
  91                 return;
  92
  93         /* we set the type of cancelation for this thread */
  94         /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
  95
  96         while (1) {
  97                 /* Make sure the p_bo object is still relevant */
  98                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
  99                         return;
 100
 101                 /* we read one mad at a time and pass it to the read callback function */
 102                 ts_ret_code =
 103                     read(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->
 104                                                             p_transp_mgr))->
 105                          device_fd, &mad, sizeof(mad));
 106                 /* Make sure the p_bo object is still relevant */
 107                 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 108                         return;
 109
 110                 if (ts_ret_code != sizeof(mad)) {
 111                         osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
 112                                 "__osmv_TOPSPIN_receiver_thr: ERR 6803: "
 113                                 "error with read, bytes = %d, errno = %d\n",
 114                                 ts_ret_code, errno);
 115                         break;
 116                 } else {
 117                         osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 118                                 "__osmv_TOPSPIN_receiver_thr: "
 119                                 "MAD QPN:%d SLID:0x%04x class:0x%02x "
 120                                 "method:0x%02x attr:0x%04x status:0x%04x "
 121                                 "tid:0x%016" PRIx64 "\n",
 122                                 mad.dqpn,
 123                                 cl_ntoh16(mad.slid),
 124                                 mad.mgmt_class,
 125                                 mad.r_method,
 126                                 cl_ntoh16(mad.attribute_id),
 127                                 cl_ntoh16(mad.status),
 128                                 cl_ntoh64(mad.transaction_id));
 129
 130                         /* first arrange an address */
 131                         __osmv_TOPSPIN_mad_addr_to_osm_addr(p_bo->p_vendor,
 132                                                             &mad,
 133                                                             (((ib_mad_t *) &
 134                                                               mad)->
 135                                                              mgmt_class ==
 136                                                              IB_MCLASS_SUBN_LID)
 137                                                             ||
 138                                                             (((ib_mad_t *) &
 139                                                               mad)->
 140                                                              mgmt_class ==
 141                                                              IB_MCLASS_SUBN_DIR),
 142                                                             &mad_addr);
 143
 144                         /* call the receiver callback */
 145
 146                         status =
 147                             osmv_dispatch_mad((osm_bind_handle_t) p_bo,
 148                                               (void *)&mad, &mad_addr);
 149
 150                         /* Make sure the p_bo object is still relevant */
 151                         if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 152                                 return;
 153
 154                         if (IB_INTERRUPTED == status) {
 155
 156                                 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 157                                         "__osmv_TOPSPIN_receiver_thr: "
 158                                         "The bind handle %p is being closed. "
 159                                         "Breaking the loop.\n", p_bo);
 160                                 break;
 161                         }
 162                 }
 163         }
 164
 165         OSM_LOG_EXIT(p_bo->p_vendor->p_log);
 166 }
 167
 168 /*
 169  * NAME
 170  *   osmv_transport_init
 171  *
 172  * DESCRIPTION
 173  *   Setup the MAD transport infrastructure (filters, callbacks etc).
 174  */
 175
 176 ib_api_status_t
 177 osmv_transport_init(IN osm_bind_info_t * p_info,
 178                     IN char hca_id[VENDOR_HCA_MAXNAMES],
 179                     IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
 180 {
 181         cl_status_t cl_st;
 182         char device_file[16];
 183         int device_fd;
 184         int ts_ioctl_ret;
 185         osmv_TOPSPIN_transport_mgr_t *p_mgr =
 186             malloc(sizeof(osmv_TOPSPIN_transport_mgr_t));
 187         int qpn;
 188
 189         if (!p_mgr) {
 190                 return IB_INSUFFICIENT_MEMORY;
 191         }
 192
 193         memset(p_mgr, 0, sizeof(osmv_TOPSPIN_transport_mgr_t));
 194
 195         /* open TopSpin file device */
 196         /* HACK: assume last char in hostid is the HCA index */
 197         sprintf(device_file, "/dev/ts_ua%u", hca_idx);
 198         device_fd = open(device_file, O_RDWR);
 199         if (device_fd < 0) {
 200                 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
 201                         device_file, errno);
 202                 return IB_ERROR;
 203         }
 204
 205         /*
 206          * Create the MAD filter on this file handle.
 207          */
 208
 209         p_mgr->filter.port = p_bo->port_num;
 210         p_mgr->filter.direction = TS_IB_MAD_DIRECTION_IN;
 211         p_mgr->filter.mask =
 212             TS_IB_MAD_FILTER_DIRECTION |
 213             TS_IB_MAD_FILTER_PORT |
 214             TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
 215
 216         switch (p_info->mad_class) {
 217         case IB_MCLASS_SUBN_LID:
 218         case IB_MCLASS_SUBN_DIR:
 219                 qpn = 0;
 220                 p_mgr->filter.qpn = qpn;
 221                 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_LID;
 222                 ts_ioctl_ret =
 223                     ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
 224                 if (ts_ioctl_ret < 0) {
 225                         return IB_ERROR;
 226                 }
 227
 228                 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_DIR;
 229                 ts_ioctl_ret =
 230                     ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
 231                 if (ts_ioctl_ret < 0) {
 232                         return IB_ERROR;
 233                 }
 234
 235                 break;
 236
 237         case IB_MCLASS_SUBN_ADM:
 238         default:
 239                 qpn = 1;
 240                 p_mgr->filter.qpn = qpn;
 241                 p_mgr->filter.mgmt_class = p_info->mad_class;
 242                 ts_ioctl_ret =
 243                     ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
 244                 if (ts_ioctl_ret < 0) {
 245                         return IB_ERROR;
 246                 }
 247                 break;
 248         }
 249
 250         p_mgr->device_fd = device_fd;
 251
 252         p_bo->p_transp_mgr = p_mgr;
 253
 254         /* Initialize the magic_ptr to the pointer of the p_bo info.
 255            This will be used to signal when the object is being destroyed, so no
 256            real action will be done then. */
 257         p_bo->magic_ptr = p_bo;
 258
 259         /* init receiver thread */
 260         cl_st =
 261             cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_receiver_thr,
 262                            (void *)p_bo, "osmv TOPSPIN rcv thr");
 263
 264         return (ib_api_status_t) cl_st;
 265 }
 266
 267 /*
 268  * NAME
 269  *   osmv_transport_send_mad
 270  *
 271  * DESCRIPTION
 272  *   Send a single MAD (256 byte)
 273  */
 274
 275 ib_api_status_t
 276 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
 277                         IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
 278 {
 279
 280         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 281         osm_vendor_t const *p_vend = p_bo->p_vendor;
 282         struct ib_mad ts_mad;
 283         int ret;
 284         ib_api_status_t status;
 285
 286         const ib_mad_t *p_mad_hdr = p_mad;
 287
 288         OSM_LOG_ENTER(p_vend->p_log);
 289
 290         memset(&ts_mad, 0, sizeof(ts_mad));
 291
 292         /* Make sure the p_bo object is still relevant */
 293         if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
 294                 return IB_INVALID_CALLBACK;
 295
 296         /*
 297          * Copy the MAD over to the sent mad
 298          */
 299         memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
 300
 301         /*
 302          * For all sends other than directed route SM MADs,
 303          * acquire an address vector for the destination.
 304          */
 305         if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
 306
 307                 __osmv_TOPSPIN_osm_addr_to_mad_addr(p_mad_addr,
 308                                                     p_mad_hdr->mgmt_class ==
 309                                                     IB_MCLASS_SUBN_LID,
 310                                                     &ts_mad);
 311         } else {
 312                 /* is a directed route - we need to construct a permissive address */
 313                 /* we do not need port number since it is part of the mad_hndl */
 314                 ts_mad.dlid = IB_LID_PERMISSIVE;
 315                 ts_mad.slid = IB_LID_PERMISSIVE;
 316                 ts_mad.sqpn = 0;
 317                 ts_mad.dqpn = 0;
 318         }
 319
 320         ts_mad.port = p_bo->port_num;
 321
 322         osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
 323                 "osmv_transport_mad_send: "
 324                 "Sending QPN:%d DLID:0x%04x class:0x%02x "
 325                 "method:0x%02x attr:0x%04x status:0x%04x "
 326                 "tid:0x%016" PRIx64 "\n",
 327                 ts_mad.dqpn,
 328                 cl_ntoh16(ts_mad.dlid),
 329                 ts_mad.mgmt_class,
 330                 ts_mad.r_method,
 331                 cl_ntoh16(ts_mad.attribute_id),
 332                 cl_ntoh16(ts_mad.status), cl_ntoh64(ts_mad.transaction_id)
 333             );
 334
 335         /* send it */
 336         ret =
 337             write(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->
 338                   device_fd, &ts_mad, sizeof(ts_mad));
 339
 340         if (ret != sizeof(ts_mad)) {
 341                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
 342                         "osmv_transport_mad_send: ERR 6804: "
 343                         "Error sending mad (%d).\n", ret);
 344                 status = IB_ERROR;
 345                 goto Exit;
 346         }
 347
 348         status = IB_SUCCESS;
 349
 350 Exit:
 351         OSM_LOG_EXIT(p_vend->p_log);
 352         return (status);
 353 }
 354
 355 /*
 356    register a new mad type to the opened device file
 357    and send a mad through - the main idea is to make
 358    the filter catch it such that the read unblocks
 359 */
 360 void __osm_transport_gen_dummy_mad(osmv_bind_obj_t * p_bo)
 361 {
 362         struct ib_mad ts_mad;
 363         osmv_TOPSPIN_transport_mgr_t *p_mgr =
 364             (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
 365         struct ib_get_port_info_ioctl port_data;
 366         int ts_ioctl_ret;
 367
 368         /* prepare the mad fields following the stored filter on the bind */
 369         memset(&ts_mad, 0, sizeof(ts_mad));
 370         ts_mad.format_version = 1;
 371         ts_mad.mgmt_class = p_mgr->filter.mgmt_class;
 372         ts_mad.attribute_id = 0x2;
 373         ts_mad.class_version = 1;
 374         ts_mad.r_method = cl_ntoh16(0x2);
 375         ts_mad.port = p_bo->port_num;
 376         ts_mad.sqpn = p_mgr->filter.qpn;
 377         ts_mad.dqpn = p_mgr->filter.qpn;
 378         ts_mad.slid = 0xffff;
 379         /* we must send to our local lid ... */
 380         port_data.port = p_bo->port_num;
 381         ts_ioctl_ret = ioctl(p_mgr->device_fd, TS_IB_IOCGPORTINFO, &port_data);
 382         ts_mad.dlid = port_data.port_info.lid;
 383         ts_mad.transaction_id = 0x9999;
 384         write(p_mgr->device_fd, &ts_mad, sizeof(ts_mad));
 385 }
 386
 387 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
 388 {
 389         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 390         osmv_TOPSPIN_transport_mgr_t *p_tpot_mgr =
 391             (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
 392
 393         CL_ASSERT(p_bo);
 394
 395         /* First of all - zero out the magic_ptr, so if a callback is called -
 396            it'll know that we are currently closing down, and will not handle the
 397            mad. */
 398         p_bo->magic_ptr = 0;
 399         /* usleep(3000000); */
 400
 401         /* seems the only way to abort a blocking read is to make it read something */
 402         __osm_transport_gen_dummy_mad(p_bo);
 403         cl_thread_destroy(&(p_tpot_mgr->receiver));
 404         free(p_tpot_mgr);
 405 }
 406
 407 static void
 408 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
 409                                     IN uint8_t is_smi, OUT struct ib_mad *p_mad)
 410 {
 411
 412         /* For global destination or Multicast address: */
 413         p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
 414         p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
 415         if (is_smi) {
 416                 p_mad->sqpn = 0;
 417                 p_mad->dqpn = 0;
 418         } else {
 419                 p_mad->sqpn = 1;
 420                 p_mad->dqpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
 421         }
 422         /*
 423            HACK we limit to the first PKey Index assuming it will
 424            always be the default PKey
 425          */
 426         p_mad->pkey_index = 0;
 427 }
 428
 429 static void
 430 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
 431                                     IN struct ib_mad *p_mad,
 432                                     IN uint8_t is_smi,
 433                                     OUT osm_mad_addr_t * p_mad_addr)
 434 {
 435         p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
 436         p_mad_addr->static_rate = 0;
 437         p_mad_addr->path_bits = 0;
 438         if (is_smi) {
 439                 /* SMI */
 440                 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
 441                 p_mad_addr->addr_type.smi.port_num = p_mad->port;
 442         } else {
 443                 /* GSI */
 444                 p_mad_addr->addr_type.gsi.remote_qp = cl_ntoh32(p_mad->sqpn);
 445                 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
 446                 /*  There is a TAVOR limitation that only one P_KEY is supported per */
 447                 /*  QP - so QP1 must use IB_DEFAULT_PKEY */
 448                 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
 449                 p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
 450
 451                 p_mad_addr->addr_type.gsi.global_route = FALSE;
 452                 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
 453                 /*
 454                    if (p_mad_addr->addr_type.gsi.global_route)
 455                    {
 456                    p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
 457                    ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
 458                    p_rcv_desc->grh.traffic_class,
 459                    p_rcv_desc->grh.flow_label);
 460                    p_mad_addr->addr_type.gsi.grh_info.hop_limit =  p_rcv_desc->grh.hop_limit;
 461                    memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
 462                    &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
 463                    memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
 464                    p_rcv_desc->grh.dgid,  sizeof(ib_net64_t));
 465                    }
 466                  */
 467         }
 468 }
 469
 470 /*
 471  *  NAME            osm_vendor_set_sm
 472  *
 473  *  DESCRIPTION     Modifies the port info for the bound port to set the "IS_SM" bit
 474  *                  according to the value given (TRUE or FALSE).
 475  */
 476 #if (defined(OSM_VENDOR_INTF_TS_NO_VAPI) || defined(OSM_VENDOR_INTF_TS))
 477
 478 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
 479 {
 480         osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
 481         osm_vendor_t const *p_vend = p_bo->p_vendor;
 482         int ts_ioctl_ret;
 483         int device_fd =
 484             ((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->device_fd;
 485         struct ib_set_port_info_ioctl set_port_data;
 486
 487         OSM_LOG_ENTER(p_vend->p_log);
 488
 489         memset(&set_port_data, 0, sizeof(set_port_data));
 490
 491         set_port_data.port = p_bo->port_num;
 492         set_port_data.port_info.valid_fields = IB_PORT_IS_SM;
 493         set_port_data.port_info.is_sm = is_sm_val;
 494         ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSPORTINFO, &set_port_data);
 495         if (ts_ioctl_ret < 0) {
 496                 osm_log(p_vend->p_log, OSM_LOG_ERROR,
 497                         "osm_vendor_set_sm: ERR 6805: "
 498                         "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
 499                         is_sm_val, ts_ioctl_ret);
 500         }
 501
 502         OSM_LOG_EXIT(p_vend->p_log);
 503 }
 504
 505 #endif