2 * Copyright (c) 2006 Voltaire, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <infiniband/umad.h>
40 #include <infiniband/mad.h>
42 #define info(fmt, arg...) fprintf(stderr, "INFO: " fmt, ##arg )
43 #define err(fmt, arg...) fprintf(stderr, "ERR: " fmt, ##arg )
45 #define dbg(fmt, arg...) fprintf(stderr, "DBG: " fmt, ##arg )
47 #define dbg(fmt, arg...)
52 /* Multicast Member Record Component Masks */
53 #define IB_MCR_COMPMASK_MGID (1ULL<<0)
54 #define IB_MCR_COMPMASK_PORT_GID (1ULL<<1)
55 #define IB_MCR_COMPMASK_QKEY (1ULL<<2)
56 #define IB_MCR_COMPMASK_MLID (1ULL<<3)
57 #define IB_MCR_COMPMASK_MTU_SEL (1ULL<<4)
58 #define IB_MCR_COMPMASK_MTU (1ULL<<5)
59 #define IB_MCR_COMPMASK_TCLASS (1ULL<<6)
60 #define IB_MCR_COMPMASK_PKEY (1ULL<<7)
61 #define IB_MCR_COMPMASK_RATE_SEL (1ULL<<8)
62 #define IB_MCR_COMPMASK_RATE (1ULL<<9)
63 #define IB_MCR_COMPMASK_LIFE_SEL (1ULL<<10)
64 #define IB_MCR_COMPMASK_LIFE (1ULL<<11)
65 #define IB_MCR_COMPMASK_SL (1ULL<<12)
66 #define IB_MCR_COMPMASK_FLOW (1ULL<<13)
67 #define IB_MCR_COMPMASK_HOP (1ULL<<14)
68 #define IB_MCR_COMPMASK_SCOPE (1ULL<<15)
69 #define IB_MCR_COMPMASK_JOIN_STATE (1ULL<<16)
70 #define IB_MCR_COMPMASK_PROXY (1ULL<<17)
72 static ibmad_gid_t mgid_ipoib = {
73 0xff, 0x12, 0x40, 0x1b, 0xff, 0xff, 0x00, 0x00,
74 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
77 uint64_t build_mcm_rec(uint8_t *data, ibmad_gid_t mgid, ibmad_gid_t port_gid)
79 memset(data, 0, IB_SA_DATA_SIZE);
80 mad_set_array(data, 0, IB_SA_MCM_MGID_F, mgid);
81 mad_set_array(data, 0, IB_SA_MCM_PORTGID_F, port_gid);
82 mad_set_field(data, 0, IB_SA_MCM_JOIN_STATE_F, 1);
84 return IB_MCR_COMPMASK_MGID|IB_MCR_COMPMASK_PORT_GID|
85 IB_MCR_COMPMASK_JOIN_STATE;
88 static void build_mcm_rec_umad(void *umad, ib_portid_t *dport, int method,
89 uint64_t comp_mask, uint8_t *data)
93 memset(&rpc, 0, sizeof(rpc));
94 rpc.mgtclass = IB_SA_CLASS;
96 rpc.attr.id = IB_SA_ATTR_MCRECORD;
97 rpc.attr.mod = 0; // ???
99 rpc.datasz = IB_SA_DATA_SIZE;
100 rpc.dataoffs = IB_SA_DATA_OFFS;
102 mad_build_pkt(umad, &rpc, dport, NULL, data);
105 static int rereg_send(int port, int agent, ib_portid_t *dport,
106 uint8_t *umad, int len, int method, ibmad_gid_t port_gid)
108 uint8_t data[IB_SA_DATA_SIZE];
111 comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
113 build_mcm_rec_umad(umad, dport, method, comp_mask, data);
114 if(umad_send(port, agent, umad, len, TMO, 0) < 0) {
115 err("umad_send leave failed: %s\n", strerror(errno));
118 dbg("umad_send %d: tid = 0x%016" PRIx64 "\n", method,
119 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
124 static int rereg_port_gid(int port, int agent, ib_portid_t *dport,
125 uint8_t *umad, int len, ibmad_gid_t port_gid)
127 uint8_t data[IB_SA_DATA_SIZE];
130 comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
132 build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_DELETE,
134 if(umad_send(port, agent, umad, len, TMO, 0) < 0) {
135 err("umad_send leave failed: %s\n", strerror(errno));
138 dbg("umad_send leave: tid = 0x%016" PRIx64 "\n",
139 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
141 build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_SET,
143 if(umad_send(port, agent, umad, len, TMO, 0) < 0) {
144 err("umad_send join failed: %s\n", strerror(errno));
147 dbg("umad_send join: tid = 0x%016" PRIx64 "\n",
148 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
159 static int rereg_send_all(int port, int agent, ib_portid_t *dport,
160 struct guid_trid *list, unsigned cnt)
163 int len = umad_size() + 256;
166 info("rereg_send_all... cnt = %u\n", cnt);
168 umad = calloc(1, len);
170 err("cannot alloc mem for umad: %s\n", strerror(errno));
174 for (i = 0; i < cnt; i++) {
175 ret = rereg_port_gid(port, agent, dport, umad, len, list[i].gid);
177 err("rereg_send_all: rereg_port_gid 0x%016" PRIx64
178 " failed\n", list[i].guid);
181 list[i].trid = mad_get_field64(umad_get_mad(umad), 0,
185 info("rereg_send_all: sent %u requests\n", cnt*2);
193 static int rereg_mcm_rec_send(int port, int agent, ib_portid_t *dport, int cnt)
196 ibmad_gid_t port_gid;
200 ib_resolve_self(&portid, NULL, &port_gid);
202 len = umad_size() + 256;
203 umad = calloc(1, len);
205 err("cannot alloc mem for umad: %s\n", strerror(errno));
210 if (!rereg_port_gid(port, agent, dport, umad, len, port_gid))
220 static int rereg_recv(int port, int agent, ib_portid_t *dport,
221 uint8_t *umad, int length, int tmo)
226 while((ret = umad_recv(port, umad, &len, tmo)) < 0 &&
227 errno == ETIMEDOUT) {
232 err("umad_recv %d failed: %s\n", ret, strerror(errno));
235 dbg("umad_recv (retries %d), tid = 0x%016" PRIx64 ": len = %d, status = %d\n",
237 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F),
238 len, umad_status(umad));
243 static int rereg_recv_all(int port, int agent, ib_portid_t *dport,
244 struct guid_trid *list, unsigned cnt)
247 int len = umad_size() + 256;
249 unsigned n, method, status;
252 info("rereg_recv_all...\n");
254 umad = calloc(1, len);
256 err("cannot alloc mem for umad: %s\n", strerror(errno));
261 while (rereg_recv(port, agent, dport, umad, len, TMO) > 0) {
262 dbg("rereg_recv_all: done %d\n", n);
264 mad = umad_get_mad(umad);
266 method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
267 status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
270 dbg("MAD status %x, method %x\n", status, method);
273 (method&0x7f) == (IB_MAD_METHOD_GET_RESPONSE&0x7f)) {
274 trid = mad_get_field64(mad, 0, IB_MAD_TRID_F);
275 for (i = 0; i < cnt; i++)
276 if (trid == list[i].trid)
279 err("cannot find trid 0x%016" PRIx64 "\n",
283 info("guid 0x%016" PRIx64 ": method = %x status = %x. Resending\n",
284 ntohll(list[i].guid), method, status);
285 rereg_port_gid(port, agent, dport, umad, len,
287 list[i].trid = mad_get_field64(umad_get_mad(umad), 0,
292 info("rereg_recv_all: got %u responses\n", n);
298 static int rereg_query_all(int port, int agent, ib_portid_t *dport,
299 struct guid_trid *list, unsigned cnt)
302 int len = umad_size() + 256;
303 unsigned method, status;
306 info("rereg_query_all...\n");
308 umad = calloc(1, len);
310 err("cannot alloc mem for umad: %s\n", strerror(errno));
314 for ( i = 0; i < cnt; i++ ) {
315 ret = rereg_send(port, agent, dport, umad, len,
316 IB_MAD_METHOD_GET, list[i].gid);
318 err("query_all: rereg_send failed.\n");
322 ret = rereg_recv(port, agent, dport, umad, len, TMO);
324 err("query_all: rereg_recv failed.\n");
328 mad = umad_get_mad(umad);
330 method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
331 status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
334 info("guid 0x%016" PRIx64 ": status %x, method %x\n",
335 ntohll(list[i].guid), status, method);
338 info("rereg_query_all: %u queried.\n", cnt);
345 static int rereg_mcm_rec_recv(int port, int agent, int cnt)
348 int len = umad_size() + 256;
351 umad = calloc(1, len);
353 err("cannot alloc mem for umad: %s\n", strerror(errno));
357 for ( i = 0; i < cnt; i++ ) {
360 while (umad_recv(port, umad, &len, TMO) < 0 &&
363 err("umad_recv %d failed: %s\n",
368 dbg("umad_recv %d (retries %d), tid = 0x%016" PRIx64 ": len = %d, status = %d\n",
370 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F),
371 len, umad_status(umad));
372 mad = umad_get_mad(umad);
380 #define MAX_CLIENTS 50
382 static int rereg_and_test_port(char *guid_file, int port, int agent, ib_portid_t *dport, int timeout)
386 ibmad_gid_t port_gid;
387 uint64_t prefix = htonll(0xfe80000000000000llu);
388 uint64_t guid = htonll(0x0002c90200223825llu);
389 struct guid_trid *list;
392 list = calloc(MAX_CLIENTS, sizeof(*list));
394 err("cannot alloc mem for guid/trid list: %s\n", strerror(errno));
398 f = fopen(guid_file, "r");
400 err("cannot open %s: %s\n", guid_file, strerror(errno));
404 while (fgets(line, sizeof(line), f)) {
405 guid = strtoull(line, NULL, 0);
407 memcpy(&port_gid[0], &prefix, 8);
408 memcpy(&port_gid[8], &guid, 8);
411 memcpy(list[i].gid, port_gid, sizeof(list[i].gid));
413 if (++i >= MAX_CLIENTS)
418 rereg_send_all(port, agent, dport, list, i);
419 rereg_recv_all(port, agent, dport, list, i);
421 rereg_query_all(port, agent, dport, list, i);
427 int main(int argc, char **argv)
429 char *guid_file = "port_guids.list";
430 int mgmt_classes[2] = {IB_SMI_CLASS, IB_SMI_DIRECT_CLASS};
431 ib_portid_t dport_id;
439 madrpc_init(NULL, 0, mgmt_classes, 2);
442 ib_resolve_smlid(&dport_id, TMO);
444 memset(&dport_id, 0, sizeof(dport_id));
449 dport_id.qkey = IB_DEFAULT_QP1_QKEY;
452 len = umad_size() + 256;
453 umad = calloc(1, len);
455 err("cannot alloc mem for umad: %s\n", strerror(errno));
460 port = madrpc_portid();
464 port = umad_open_port(NULL, 0);
466 err("umad_open_port failed: %s\n", strerror(errno));
471 agent = umad_register(port, IB_SA_CLASS, 2, 0, NULL);
475 cnt = rereg_mcm_rec_send(port, agent, &dport_id, cnt);
477 rereg_recv_all(port, agent, &dport_id);
479 rereg_and_test_port(guid_file, port, agent, &dport_id, TMO);
481 mad = umad_get_mad(umad);
484 umad_unregister(port, agent);
485 umad_close_port(port);