2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 /* AUTHOR Edward Bortnikov
39 * The lower-level MAD transport interface implementation
40 * that allows sending a single MAD/receiving a callback
41 * when a single MAD is received.
46 #endif /* HAVE_CONFIG_H */
48 #include <sys/types.h>
50 #include <sys/ioctl.h>
56 #include <vendor/osm_vendor_api.h>
57 #include <vendor/osm_vendor_mlx_transport.h>
58 #include <vendor/osm_vendor_mlx_dispatcher.h>
59 #include <vendor/osm_vendor_mlx_svc.h>
60 #include <vendor/osm_ts_useraccess.h>
62 typedef struct _osmv_TOPSPIN_transport_mgr_ {
64 osm_ts_user_mad_filter filter;
66 } osmv_TOPSPIN_transport_mgr_t;
69 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
70 IN struct ib_mad *p_mad,
72 OUT osm_mad_addr_t * p_mad_addr);
75 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
77 OUT struct ib_mad *p_mad);
79 void __osmv_TOPSPIN_receiver_thr(void *p_ctx)
83 osm_mad_addr_t mad_addr;
84 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
85 ib_api_status_t status = IB_SUCCESS;
87 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
89 /* Make sure the p_bo object is still relevant */
90 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
93 /* we set the type of cancelation for this thread */
94 /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
97 /* Make sure the p_bo object is still relevant */
98 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
101 /* we read one mad at a time and pass it to the read callback function */
103 read(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->
105 device_fd, &mad, sizeof(mad));
106 /* Make sure the p_bo object is still relevant */
107 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
110 if (ts_ret_code != sizeof(mad)) {
111 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
112 "__osmv_TOPSPIN_receiver_thr: ERR 6803: "
113 "error with read, bytes = %d, errno = %d\n",
117 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
118 "__osmv_TOPSPIN_receiver_thr: "
119 "MAD QPN:%d SLID:0x%04x class:0x%02x "
120 "method:0x%02x attr:0x%04x status:0x%04x "
121 "tid:0x%016" PRIx64 "\n",
126 cl_ntoh16(mad.attribute_id),
127 cl_ntoh16(mad.status),
128 cl_ntoh64(mad.transaction_id));
130 /* first arrange an address */
131 __osmv_TOPSPIN_mad_addr_to_osm_addr(p_bo->p_vendor,
144 /* call the receiver callback */
147 osmv_dispatch_mad((osm_bind_handle_t) p_bo,
148 (void *)&mad, &mad_addr);
150 /* Make sure the p_bo object is still relevant */
151 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
154 if (IB_INTERRUPTED == status) {
156 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
157 "__osmv_TOPSPIN_receiver_thr: "
158 "The bind handle %p is being closed. "
159 "Breaking the loop.\n", p_bo);
165 OSM_LOG_EXIT(p_bo->p_vendor->p_log);
170 * osmv_transport_init
173 * Setup the MAD transport infrastructure (filters, callbacks etc).
177 osmv_transport_init(IN osm_bind_info_t * p_info,
178 IN char hca_id[VENDOR_HCA_MAXNAMES],
179 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
182 char device_file[16];
185 osmv_TOPSPIN_transport_mgr_t *p_mgr =
186 malloc(sizeof(osmv_TOPSPIN_transport_mgr_t));
190 return IB_INSUFFICIENT_MEMORY;
193 memset(p_mgr, 0, sizeof(osmv_TOPSPIN_transport_mgr_t));
195 /* open TopSpin file device */
196 /* HACK: assume last char in hostid is the HCA index */
197 sprintf(device_file, "/dev/ts_ua%u", hca_idx);
198 device_fd = open(device_file, O_RDWR);
200 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
206 * Create the MAD filter on this file handle.
209 p_mgr->filter.port = p_bo->port_num;
210 p_mgr->filter.direction = TS_IB_MAD_DIRECTION_IN;
212 TS_IB_MAD_FILTER_DIRECTION |
213 TS_IB_MAD_FILTER_PORT |
214 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
216 switch (p_info->mad_class) {
217 case IB_MCLASS_SUBN_LID:
218 case IB_MCLASS_SUBN_DIR:
220 p_mgr->filter.qpn = qpn;
221 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_LID;
223 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
224 if (ts_ioctl_ret < 0) {
228 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_DIR;
230 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
231 if (ts_ioctl_ret < 0) {
237 case IB_MCLASS_SUBN_ADM:
240 p_mgr->filter.qpn = qpn;
241 p_mgr->filter.mgmt_class = p_info->mad_class;
243 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter);
244 if (ts_ioctl_ret < 0) {
250 p_mgr->device_fd = device_fd;
252 p_bo->p_transp_mgr = p_mgr;
254 /* Initialize the magic_ptr to the pointer of the p_bo info.
255 This will be used to signal when the object is being destroyed, so no
256 real action will be done then. */
257 p_bo->magic_ptr = p_bo;
259 /* init receiver thread */
261 cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_receiver_thr,
262 (void *)p_bo, "osmv TOPSPIN rcv thr");
264 return (ib_api_status_t) cl_st;
269 * osmv_transport_send_mad
272 * Send a single MAD (256 byte)
276 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
277 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
280 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
281 osm_vendor_t const *p_vend = p_bo->p_vendor;
282 struct ib_mad ts_mad;
284 ib_api_status_t status;
286 const ib_mad_t *p_mad_hdr = p_mad;
288 OSM_LOG_ENTER(p_vend->p_log);
290 memset(&ts_mad, 0, sizeof(ts_mad));
292 /* Make sure the p_bo object is still relevant */
293 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
294 return IB_INVALID_CALLBACK;
297 * Copy the MAD over to the sent mad
299 memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
302 * For all sends other than directed route SM MADs,
303 * acquire an address vector for the destination.
305 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
307 __osmv_TOPSPIN_osm_addr_to_mad_addr(p_mad_addr,
308 p_mad_hdr->mgmt_class ==
312 /* is a directed route - we need to construct a permissive address */
313 /* we do not need port number since it is part of the mad_hndl */
314 ts_mad.dlid = IB_LID_PERMISSIVE;
315 ts_mad.slid = IB_LID_PERMISSIVE;
320 ts_mad.port = p_bo->port_num;
322 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
323 "osmv_transport_mad_send: "
324 "Sending QPN:%d DLID:0x%04x class:0x%02x "
325 "method:0x%02x attr:0x%04x status:0x%04x "
326 "tid:0x%016" PRIx64 "\n",
328 cl_ntoh16(ts_mad.dlid),
331 cl_ntoh16(ts_mad.attribute_id),
332 cl_ntoh16(ts_mad.status), cl_ntoh64(ts_mad.transaction_id)
337 write(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->
338 device_fd, &ts_mad, sizeof(ts_mad));
340 if (ret != sizeof(ts_mad)) {
341 osm_log(p_vend->p_log, OSM_LOG_ERROR,
342 "osmv_transport_mad_send: ERR 6804: "
343 "Error sending mad (%d).\n", ret);
351 OSM_LOG_EXIT(p_vend->p_log);
356 register a new mad type to the opened device file
357 and send a mad through - the main idea is to make
358 the filter catch it such that the read unblocks
360 void __osm_transport_gen_dummy_mad(osmv_bind_obj_t * p_bo)
362 struct ib_mad ts_mad;
363 osmv_TOPSPIN_transport_mgr_t *p_mgr =
364 (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
365 struct ib_get_port_info_ioctl port_data;
368 /* prepare the mad fields following the stored filter on the bind */
369 memset(&ts_mad, 0, sizeof(ts_mad));
370 ts_mad.format_version = 1;
371 ts_mad.mgmt_class = p_mgr->filter.mgmt_class;
372 ts_mad.attribute_id = 0x2;
373 ts_mad.class_version = 1;
374 ts_mad.r_method = cl_ntoh16(0x2);
375 ts_mad.port = p_bo->port_num;
376 ts_mad.sqpn = p_mgr->filter.qpn;
377 ts_mad.dqpn = p_mgr->filter.qpn;
378 ts_mad.slid = 0xffff;
379 /* we must send to our local lid ... */
380 port_data.port = p_bo->port_num;
381 ts_ioctl_ret = ioctl(p_mgr->device_fd, TS_IB_IOCGPORTINFO, &port_data);
382 ts_mad.dlid = port_data.port_info.lid;
383 ts_mad.transaction_id = 0x9999;
384 write(p_mgr->device_fd, &ts_mad, sizeof(ts_mad));
387 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
389 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
390 osmv_TOPSPIN_transport_mgr_t *p_tpot_mgr =
391 (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr);
395 /* First of all - zero out the magic_ptr, so if a callback is called -
396 it'll know that we are currently closing down, and will not handle the
399 /* usleep(3000000); */
401 /* seems the only way to abort a blocking read is to make it read something */
402 __osm_transport_gen_dummy_mad(p_bo);
403 cl_thread_destroy(&(p_tpot_mgr->receiver));
408 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
409 IN uint8_t is_smi, OUT struct ib_mad *p_mad)
412 /* For global destination or Multicast address: */
413 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
414 p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
420 p_mad->dqpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
423 HACK we limit to the first PKey Index assuming it will
424 always be the default PKey
426 p_mad->pkey_index = 0;
430 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
431 IN struct ib_mad *p_mad,
433 OUT osm_mad_addr_t * p_mad_addr)
435 p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
436 p_mad_addr->static_rate = 0;
437 p_mad_addr->path_bits = 0;
440 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
441 p_mad_addr->addr_type.smi.port_num = p_mad->port;
444 p_mad_addr->addr_type.gsi.remote_qp = cl_ntoh32(p_mad->sqpn);
445 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
446 /* There is a TAVOR limitation that only one P_KEY is supported per */
447 /* QP - so QP1 must use IB_DEFAULT_PKEY */
448 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
449 p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
451 p_mad_addr->addr_type.gsi.global_route = FALSE;
452 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
454 if (p_mad_addr->addr_type.gsi.global_route)
456 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
457 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
458 p_rcv_desc->grh.traffic_class,
459 p_rcv_desc->grh.flow_label);
460 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
461 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
462 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
463 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
464 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
471 * NAME osm_vendor_set_sm
473 * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit
474 * according to the value given (TRUE or FALSE).
476 #if (defined(OSM_VENDOR_INTF_TS_NO_VAPI) || defined(OSM_VENDOR_INTF_TS))
478 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
480 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
481 osm_vendor_t const *p_vend = p_bo->p_vendor;
484 ((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->device_fd;
485 struct ib_set_port_info_ioctl set_port_data;
487 OSM_LOG_ENTER(p_vend->p_log);
489 memset(&set_port_data, 0, sizeof(set_port_data));
491 set_port_data.port = p_bo->port_num;
492 set_port_data.port_info.valid_fields = IB_PORT_IS_SM;
493 set_port_data.port_info.is_sm = is_sm_val;
494 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSPORTINFO, &set_port_data);
495 if (ts_ioctl_ret < 0) {
496 osm_log(p_vend->p_log, OSM_LOG_ERROR,
497 "osm_vendor_set_sm: ERR 6805: "
498 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
499 is_sm_val, ts_ioctl_ret);
502 OSM_LOG_EXIT(p_vend->p_log);