1 [PATCH 4/4] libibverbs: Update examples
3 Since RDMAoE requires usage of GRH, update ibv_*_pinpong examples to accept
4 GIDs. GIDs are given as an index to the local port's table and are exchanged
5 between the client and the server through the socket connection.
7 Signed-off-by: Eli Cohen <eli@mellanox.co.il>
9 examples/devinfo.c | 14 ++++++++
10 examples/pingpong.c | 31 ++++++++++++++++++
11 examples/pingpong.h | 4 ++
12 examples/rc_pingpong.c | 79 +++++++++++++++++++++++++++++++++++-----------
13 examples/srq_pingpong.c | 72 ++++++++++++++++++++++++++++++++----------
14 examples/uc_pingpong.c | 70 ++++++++++++++++++++++++++++++++---------
15 examples/ud_pingpong.c | 69 ++++++++++++++++++++++++++++++-----------
16 7 files changed, 269 insertions(+), 70 deletions(-)
18 Index: libibverbs/examples/devinfo.c
19 ===================================================================
20 --- libibverbs.orig/examples/devinfo.c 2010-02-08 15:04:24.369329000 +0200
21 +++ libibverbs/examples/devinfo.c 2010-03-17 14:08:48.404754000 +0200
22 @@ -184,6 +184,19 @@ static int print_all_port_gids(struct ib
26 +static const char *link_layer_str(uint8_t link_layer)
28 + switch (link_layer) {
29 + case IBV_LINK_LAYER_UNSPECIFIED:
30 + case IBV_LINK_LAYER_INFINIBAND:
32 + case IBV_LINK_LAYER_ETHERNET:
39 static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
41 struct ibv_context *ctx;
42 @@ -284,6 +297,7 @@ static int print_hca_cap(struct ibv_devi
43 printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid);
44 printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid);
45 printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc);
46 + printf("\t\t\tlink_layer:\t\t%s\n", link_layer_str(port_attr.link_layer));
49 printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz);
50 Index: libibverbs/examples/pingpong.c
51 ===================================================================
52 --- libibverbs.orig/examples/pingpong.c 2010-02-08 15:04:24.372330000 +0200
53 +++ libibverbs/examples/pingpong.c 2010-03-17 14:08:48.433754000 +0200
58 +#include <arpa/inet.h>
63 enum ibv_mtu pp_mtu_to_enum(int mtu)
65 @@ -53,3 +57,30 @@ uint16_t pp_get_local_lid(struct ibv_con
70 +int pp_get_port_info(struct ibv_context *context, int port,
71 + struct ibv_port_attr *attr)
73 + return ibv_query_port(context, port, attr);
76 +void wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
82 + for (tmp[8] = 0, i = 0; i < 4; ++i) {
83 + memcpy(tmp, wgid + i * 8, 8);
84 + sscanf(tmp, "%x", &v32);
85 + *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32);
89 +void gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
93 + for (i = 0; i < 4; ++i)
94 + sprintf(&wgid[i * 8], "%08x", htonl(*(uint32_t *)(gid->raw + i * 4)));
96 Index: libibverbs/examples/pingpong.h
97 ===================================================================
98 --- libibverbs.orig/examples/pingpong.h 2010-02-08 15:04:24.375328000 +0200
99 +++ libibverbs/examples/pingpong.h 2010-03-17 14:08:48.443756000 +0200
102 enum ibv_mtu pp_mtu_to_enum(int mtu);
103 uint16_t pp_get_local_lid(struct ibv_context *context, int port);
104 +int pp_get_port_info(struct ibv_context *context, int port,
105 + struct ibv_port_attr *attr);
106 +void wire_gid_to_gid(const char *wgid, union ibv_gid *gid);
107 +void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]);
109 #endif /* IBV_PINGPONG_H */
110 Index: libibverbs/examples/rc_pingpong.c
111 ===================================================================
112 --- libibverbs.orig/examples/rc_pingpong.c 2010-02-08 15:04:24.378329000 +0200
113 +++ libibverbs/examples/rc_pingpong.c 2010-03-17 14:11:16.145313000 +0200
114 @@ -67,17 +67,19 @@ struct pingpong_context {
118 + struct ibv_port_attr portinfo;
121 struct pingpong_dest {
128 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
129 enum ibv_mtu mtu, int sl,
130 - struct pingpong_dest *dest)
131 + struct pingpong_dest *dest, int sgid_idx)
133 struct ibv_qp_attr attr = {
134 .qp_state = IBV_QPS_RTR,
135 @@ -94,6 +96,13 @@ static int pp_connect_ctx(struct pingpon
140 + if (dest->gid.global.interface_id) {
141 + attr.ah_attr.is_global = 1;
142 + attr.ah_attr.grh.hop_limit = 1;
143 + attr.ah_attr.grh.dgid = dest->gid;
144 + attr.ah_attr.grh.sgid_index = sgid_idx;
146 if (ibv_modify_qp(ctx->qp, &attr,
149 @@ -135,10 +144,11 @@ static struct pingpong_dest *pp_client_e
150 .ai_socktype = SOCK_STREAM
153 - char msg[sizeof "0000:000000:000000"];
154 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
157 struct pingpong_dest *rem_dest = NULL;
160 if (asprintf(&service, "%d", port) < 0)
162 @@ -169,7 +179,8 @@ static struct pingpong_dest *pp_client_e
166 - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
167 + gid_to_wire_gid(&my_dest->gid, gid);
168 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
169 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
170 fprintf(stderr, "Couldn't send local address\n");
172 @@ -187,7 +198,8 @@ static struct pingpong_dest *pp_client_e
176 - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
177 + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
178 + wire_gid_to_gid(gid, &rem_dest->gid);
182 @@ -197,7 +209,8 @@ out:
183 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
184 int ib_port, enum ibv_mtu mtu,
186 - const struct pingpong_dest *my_dest)
187 + const struct pingpong_dest *my_dest,
190 struct addrinfo *res, *t;
191 struct addrinfo hints = {
192 @@ -206,10 +219,11 @@ static struct pingpong_dest *pp_server_e
193 .ai_socktype = SOCK_STREAM
196 - char msg[sizeof "0000:000000:000000"];
197 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
199 int sockfd = -1, connfd;
200 struct pingpong_dest *rem_dest = NULL;
203 if (asprintf(&service, "%d", port) < 0)
205 @@ -263,16 +277,19 @@ static struct pingpong_dest *pp_server_e
209 - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
210 + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
211 + wire_gid_to_gid(gid, &rem_dest->gid);
213 - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) {
214 + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) {
215 fprintf(stderr, "Couldn't connect to remote QP\n");
221 - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
223 + gid_to_wire_gid(&my_dest->gid, gid);
224 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
225 if (write(connfd, msg, sizeof msg) != sizeof msg) {
226 fprintf(stderr, "Couldn't send local address\n");
228 @@ -289,11 +306,11 @@ out:
230 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
231 int rx_depth, int port,
233 + int use_event, int is_server)
235 struct pingpong_context *ctx;
237 - ctx = malloc(sizeof *ctx);
238 + ctx = calloc(1, sizeof *ctx);
242 @@ -306,7 +323,7 @@ static struct pingpong_context *pp_init_
246 - memset(ctx->buf, 0, size);
247 + memset(ctx->buf, 0x7b + is_server, size);
249 ctx->context = ibv_open_device(ib_dev);
251 @@ -481,6 +498,7 @@ static void usage(const char *argv0)
252 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
253 printf(" -l, --sl=<sl> service level value\n");
254 printf(" -e, --events sleep on CQ events (default poll)\n");
255 + printf(" -g, --gid-idx=<gid index> local port gid index\n");
258 int main(int argc, char *argv[])
259 @@ -504,6 +522,8 @@ int main(int argc, char *argv[])
261 int num_cq_events = 0;
266 srand48(getpid() * time(NULL));
268 @@ -520,10 +540,11 @@ int main(int argc, char *argv[])
269 { .name = "iters", .has_arg = 1, .val = 'n' },
270 { .name = "sl", .has_arg = 1, .val = 'l' },
271 { .name = "events", .has_arg = 0, .val = 'e' },
272 + { .name = "gid-idx", .has_arg = 1, .val = 'g' },
276 - c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL);
277 + c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL);
281 @@ -576,6 +597,10 @@ int main(int argc, char *argv[])
286 + gidx = strtol(optarg, NULL, 0);
292 @@ -615,7 +640,7 @@ int main(int argc, char *argv[])
296 - ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
297 + ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event, !servername);
301 @@ -631,30 +656,47 @@ int main(int argc, char *argv[])
305 - my_dest.lid = pp_get_local_lid(ctx->context, ib_port);
306 - my_dest.qpn = ctx->qp->qp_num;
307 - my_dest.psn = lrand48() & 0xffffff;
308 - if (!my_dest.lid) {
310 + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
311 + fprintf(stderr, "Couldn't get port info\n");
315 + my_dest.lid = ctx->portinfo.lid;
316 + if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) {
317 fprintf(stderr, "Couldn't get local LID\n");
321 - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
322 - my_dest.lid, my_dest.qpn, my_dest.psn);
324 + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
325 + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
329 + memset(&my_dest.gid, 0, sizeof my_dest.gid);
331 + my_dest.qpn = ctx->qp->qp_num;
332 + my_dest.psn = lrand48() & 0xffffff;
333 + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
334 + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
335 + my_dest.lid, my_dest.qpn, my_dest.psn, gid);
339 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
341 - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest);
342 + rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
347 - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
348 - rem_dest->lid, rem_dest->qpn, rem_dest->psn);
349 + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
350 + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
351 + rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
354 - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest))
355 + if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx))
358 ctx->pending = PINGPONG_RECV_WRID;
359 @@ -706,6 +748,7 @@ int main(int argc, char *argv[])
360 fprintf(stderr, "poll CQ failed %d\n", ne);
364 } while (!use_event && ne < 1);
366 for (i = 0; i < ne; ++i) {
367 Index: libibverbs/examples/srq_pingpong.c
368 ===================================================================
369 --- libibverbs.orig/examples/srq_pingpong.c 2010-02-08 15:04:24.382329000 +0200
370 +++ libibverbs/examples/srq_pingpong.c 2010-03-17 14:13:22.332220000 +0200
371 @@ -71,17 +71,19 @@ struct pingpong_context {
375 + struct ibv_port_attr portinfo;
378 struct pingpong_dest {
385 static int pp_connect_ctx(struct pingpong_context *ctx, int port, enum ibv_mtu mtu,
386 int sl, const struct pingpong_dest *my_dest,
387 - const struct pingpong_dest *dest)
388 + const struct pingpong_dest *dest, int sgid_idx)
392 @@ -101,6 +103,13 @@ static int pp_connect_ctx(struct pingpon
397 + if (dest->gid.global.interface_id) {
398 + attr.ah_attr.is_global = 1;
399 + attr.ah_attr.grh.hop_limit = 1;
400 + attr.ah_attr.grh.dgid = dest->gid;
401 + attr.ah_attr.grh.sgid_index = sgid_idx;
403 if (ibv_modify_qp(ctx->qp[i], &attr,
406 @@ -143,12 +152,13 @@ static struct pingpong_dest *pp_client_e
407 .ai_socktype = SOCK_STREAM
410 - char msg[sizeof "0000:000000:000000"];
411 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
416 struct pingpong_dest *rem_dest = NULL;
419 if (asprintf(&service, "%d", port) < 0)
421 @@ -180,7 +190,8 @@ static struct pingpong_dest *pp_client_e
424 for (i = 0; i < MAX_QP; ++i) {
425 - sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn);
426 + gid_to_wire_gid(&my_dest[i].gid, gid);
427 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid);
428 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
429 fprintf(stderr, "Couldn't send local address\n");
431 @@ -204,8 +215,9 @@ static struct pingpong_dest *pp_client_e
435 - sscanf(msg, "%x:%x:%x",
436 - &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn);
437 + sscanf(msg, "%x:%x:%x:%s",
438 + &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid);
439 + wire_gid_to_gid(gid, &rem_dest[i].gid);
442 write(sockfd, "done", sizeof "done");
443 @@ -218,7 +230,8 @@ out:
444 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
445 int ib_port, enum ibv_mtu mtu,
447 - const struct pingpong_dest *my_dest)
448 + const struct pingpong_dest *my_dest,
451 struct addrinfo *res, *t;
452 struct addrinfo hints = {
453 @@ -227,12 +240,13 @@ static struct pingpong_dest *pp_server_e
454 .ai_socktype = SOCK_STREAM
457 - char msg[sizeof "0000:000000:000000"];
458 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
462 int sockfd = -1, connfd;
463 struct pingpong_dest *rem_dest = NULL;
466 if (asprintf(&service, "%d", port) < 0)
468 @@ -292,11 +306,12 @@ static struct pingpong_dest *pp_server_e
472 - sscanf(msg, "%x:%x:%x",
473 - &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn);
474 + sscanf(msg, "%x:%x:%x:%s",
475 + &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid);
476 + wire_gid_to_gid(gid, &rem_dest[i].gid);
479 - if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) {
480 + if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, sgid_idx)) {
481 fprintf(stderr, "Couldn't connect to remote QP\n");
484 @@ -304,7 +319,8 @@ static struct pingpong_dest *pp_server_e
487 for (i = 0; i < MAX_QP; ++i) {
488 - sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn);
489 + gid_to_wire_gid(&my_dest[i].gid, gid);
490 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid);
491 if (write(connfd, msg, sizeof msg) != sizeof msg) {
492 fprintf(stderr, "Couldn't send local address\n");
494 @@ -327,7 +343,7 @@ static struct pingpong_context *pp_init_
495 struct pingpong_context *ctx;
498 - ctx = malloc(sizeof *ctx);
499 + ctx = calloc(1, sizeof *ctx);
503 @@ -551,6 +567,7 @@ static void usage(const char *argv0)
504 printf(" -n, --iters=<iters> number of exchanges per QP(default 1000)\n");
505 printf(" -l, --sl=<sl> service level value\n");
506 printf(" -e, --events sleep on CQ events (default poll)\n");
507 + printf(" -g, --gid-idx=<gid index> local port gid index\n");
510 int main(int argc, char *argv[])
511 @@ -578,6 +595,8 @@ int main(int argc, char *argv[])
513 int num_cq_events = 0;
518 srand48(getpid() * time(NULL));
520 @@ -595,10 +614,11 @@ int main(int argc, char *argv[])
521 { .name = "iters", .has_arg = 1, .val = 'n' },
522 { .name = "sl", .has_arg = 1, .val = 'l' },
523 { .name = "events", .has_arg = 0, .val = 'e' },
524 + { .name = "gid-idx", .has_arg = 1, .val = 'g' },
528 - c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:e", long_options, NULL);
529 + c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:", long_options, NULL);
533 @@ -655,6 +675,10 @@ int main(int argc, char *argv[])
538 + gidx = strtol(optarg, NULL, 0);
544 @@ -722,33 +746,50 @@ int main(int argc, char *argv[])
546 memset(my_dest, 0, sizeof my_dest);
548 + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
549 + fprintf(stderr, "Couldn't get port info\n");
552 for (i = 0; i < num_qp; ++i) {
553 my_dest[i].qpn = ctx->qp[i]->qp_num;
554 my_dest[i].psn = lrand48() & 0xffffff;
555 - my_dest[i].lid = pp_get_local_lid(ctx->context, ib_port);
556 - if (!my_dest[i].lid) {
557 + my_dest[i].lid = ctx->portinfo.lid;
558 + if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest[i].lid) {
559 fprintf(stderr, "Couldn't get local LID\n");
563 - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
564 - my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn);
566 + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest[i].gid)) {
567 + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
571 + memset(&my_dest[i].gid, 0, sizeof my_dest[i].gid);
573 + inet_ntop(AF_INET6, &my_dest[i].gid, gid, sizeof gid);
574 + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
575 + my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid);
579 rem_dest = pp_client_exch_dest(servername, port, my_dest);
581 - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest);
582 + rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest, gidx);
587 - for (i = 0; i < num_qp; ++i)
588 - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
589 - rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn);
590 + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
592 + for (i = 0; i < num_qp; ++i) {
593 + inet_ntop(AF_INET6, &rem_dest[i].gid, gid, sizeof gid);
594 + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
595 + rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, gid);
599 - if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest))
600 + if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, gidx))
604 Index: libibverbs/examples/uc_pingpong.c
605 ===================================================================
606 --- libibverbs.orig/examples/uc_pingpong.c 2010-02-08 15:04:24.386328000 +0200
607 +++ libibverbs/examples/uc_pingpong.c 2010-03-17 14:14:23.573114000 +0200
608 @@ -67,17 +67,19 @@ struct pingpong_context {
612 + struct ibv_port_attr portinfo;
615 struct pingpong_dest {
622 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
623 enum ibv_mtu mtu, int sl,
624 - struct pingpong_dest *dest)
625 + struct pingpong_dest *dest, int sgid_idx)
627 struct ibv_qp_attr attr = {
628 .qp_state = IBV_QPS_RTR,
629 @@ -92,6 +94,14 @@ static int pp_connect_ctx(struct pingpon
634 + if (dest->gid.global.interface_id) {
635 + attr.ah_attr.is_global = 1;
636 + attr.ah_attr.grh.hop_limit = 1;
637 + attr.ah_attr.grh.dgid = dest->gid;
638 + attr.ah_attr.grh.sgid_index = sgid_idx;
641 if (ibv_modify_qp(ctx->qp, &attr,
644 @@ -123,10 +133,11 @@ static struct pingpong_dest *pp_client_e
645 .ai_socktype = SOCK_STREAM
648 - char msg[sizeof "0000:000000:000000"];
649 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
652 struct pingpong_dest *rem_dest = NULL;
655 if (asprintf(&service, "%d", port) < 0)
657 @@ -157,7 +168,8 @@ static struct pingpong_dest *pp_client_e
661 - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
662 + gid_to_wire_gid(&my_dest->gid, gid);
663 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
664 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
665 fprintf(stderr, "Couldn't send local address\n");
667 @@ -175,7 +187,8 @@ static struct pingpong_dest *pp_client_e
671 - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
672 + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
673 + wire_gid_to_gid(gid, &rem_dest->gid);
677 @@ -185,7 +198,8 @@ out:
678 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
679 int ib_port, enum ibv_mtu mtu,
681 - const struct pingpong_dest *my_dest)
682 + const struct pingpong_dest *my_dest,
685 struct addrinfo *res, *t;
686 struct addrinfo hints = {
687 @@ -194,10 +208,11 @@ static struct pingpong_dest *pp_server_e
688 .ai_socktype = SOCK_STREAM
691 - char msg[sizeof "0000:000000:000000"];
692 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
694 int sockfd = -1, connfd;
695 struct pingpong_dest *rem_dest = NULL;
698 if (asprintf(&service, "%d", port) < 0)
700 @@ -251,16 +266,18 @@ static struct pingpong_dest *pp_server_e
704 - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
705 + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
706 + wire_gid_to_gid(gid, &rem_dest->gid);
708 - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) {
709 + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) {
710 fprintf(stderr, "Couldn't connect to remote QP\n");
716 - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
717 + gid_to_wire_gid(&my_dest->gid, gid);
718 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
719 if (write(connfd, msg, sizeof msg) != sizeof msg) {
720 fprintf(stderr, "Couldn't send local address\n");
722 @@ -281,7 +298,7 @@ static struct pingpong_context *pp_init_
724 struct pingpong_context *ctx;
726 - ctx = malloc(sizeof *ctx);
727 + ctx = calloc(1, sizeof *ctx);
731 @@ -469,6 +486,7 @@ static void usage(const char *argv0)
732 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
733 printf(" -l, --sl=<sl> service level value\n");
734 printf(" -e, --events sleep on CQ events (default poll)\n");
735 + printf(" -g, --gid-idx=<gid index> local port gid index\n");
738 int main(int argc, char *argv[])
739 @@ -492,6 +510,8 @@ int main(int argc, char *argv[])
741 int num_cq_events = 0;
746 srand48(getpid() * time(NULL));
748 @@ -508,10 +528,11 @@ int main(int argc, char *argv[])
749 { .name = "iters", .has_arg = 1, .val = 'n' },
750 { .name = "sl", .has_arg = 1, .val = 'l' },
751 { .name = "events", .has_arg = 0, .val = 'e' },
752 + { .name = "gid-idx", .has_arg = 1, .val = 'g' },
756 - c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL);
757 + c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL);
761 @@ -564,6 +585,10 @@ int main(int argc, char *argv[])
766 + gidx = strtol(optarg, NULL, 0);
772 @@ -619,30 +644,45 @@ int main(int argc, char *argv[])
776 - my_dest.lid = pp_get_local_lid(ctx->context, ib_port);
777 - my_dest.qpn = ctx->qp->qp_num;
778 - my_dest.psn = lrand48() & 0xffffff;
779 - if (!my_dest.lid) {
780 + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
781 + fprintf(stderr, "Couldn't get port info\n");
785 + my_dest.lid = ctx->portinfo.lid;
786 + if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) {
787 fprintf(stderr, "Couldn't get local LID\n");
791 - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
792 - my_dest.lid, my_dest.qpn, my_dest.psn);
794 + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
795 + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
799 + memset(&my_dest.gid, 0, sizeof my_dest.gid);
801 + my_dest.qpn = ctx->qp->qp_num;
802 + my_dest.psn = lrand48() & 0xffffff;
803 + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
804 + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
805 + my_dest.lid, my_dest.qpn, my_dest.psn, gid);
808 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
810 - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest);
811 + rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
816 - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
817 - rem_dest->lid, rem_dest->qpn, rem_dest->psn);
818 + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
819 + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
820 + rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
823 - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest))
824 + if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx))
827 ctx->pending = PINGPONG_RECV_WRID;
828 Index: libibverbs/examples/ud_pingpong.c
829 ===================================================================
830 --- libibverbs.orig/examples/ud_pingpong.c 2010-02-08 15:04:24.389329000 +0200
831 +++ libibverbs/examples/ud_pingpong.c 2010-03-17 14:08:48.502754000 +0200
832 @@ -68,16 +68,18 @@ struct pingpong_context {
836 + struct ibv_port_attr portinfo;
839 struct pingpong_dest {
846 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
847 - int sl, struct pingpong_dest *dest)
848 + int sl, struct pingpong_dest *dest, int sgid_idx)
850 struct ibv_ah_attr ah_attr = {
852 @@ -105,6 +107,13 @@ static int pp_connect_ctx(struct pingpon
856 + if (dest->gid.global.interface_id) {
857 + ah_attr.is_global = 1;
858 + ah_attr.grh.hop_limit = 1;
859 + ah_attr.grh.dgid = dest->gid;
860 + ah_attr.grh.sgid_index = sgid_idx;
863 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
865 fprintf(stderr, "Failed to create AH\n");
866 @@ -123,10 +132,11 @@ static struct pingpong_dest *pp_client_e
867 .ai_socktype = SOCK_STREAM
870 - char msg[sizeof "0000:000000:000000"];
871 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
874 struct pingpong_dest *rem_dest = NULL;
877 if (asprintf(&service, "%d", port) < 0)
879 @@ -157,7 +167,8 @@ static struct pingpong_dest *pp_client_e
883 - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
884 + gid_to_wire_gid(&my_dest->gid, gid);
885 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
886 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
887 fprintf(stderr, "Couldn't send local address\n");
889 @@ -175,7 +186,8 @@ static struct pingpong_dest *pp_client_e
893 - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
894 + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
895 + wire_gid_to_gid(gid, &rem_dest->gid);
899 @@ -184,7 +196,8 @@ out:
901 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
902 int ib_port, int port, int sl,
903 - const struct pingpong_dest *my_dest)
904 + const struct pingpong_dest *my_dest,
907 struct addrinfo *res, *t;
908 struct addrinfo hints = {
909 @@ -193,10 +206,11 @@ static struct pingpong_dest *pp_server_e
910 .ai_socktype = SOCK_STREAM
913 - char msg[sizeof "0000:000000:000000"];
914 + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
916 int sockfd = -1, connfd;
917 struct pingpong_dest *rem_dest = NULL;
920 if (asprintf(&service, "%d", port) < 0)
922 @@ -250,16 +264,18 @@ static struct pingpong_dest *pp_server_e
926 - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn);
927 + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
928 + wire_gid_to_gid(gid, &rem_dest->gid);
930 - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest)) {
931 + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) {
932 fprintf(stderr, "Couldn't connect to remote QP\n");
938 - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn);
939 + gid_to_wire_gid(&my_dest->gid, gid);
940 + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
941 if (write(connfd, msg, sizeof msg) != sizeof msg) {
942 fprintf(stderr, "Couldn't send local address\n");
944 @@ -474,10 +490,11 @@ static void usage(const char *argv0)
945 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
946 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
947 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
948 - printf(" -s, --size=<size> size of message to exchange (default 2048)\n");
949 + printf(" -s, --size=<size> size of message to exchange (default 1024)\n");
950 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
951 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
952 printf(" -e, --events sleep on CQ events (default poll)\n");
953 + printf(" -g, --gid-idx=<gid index> local port gid index\n");
956 int main(int argc, char *argv[])
957 @@ -492,7 +509,7 @@ int main(int argc, char *argv[])
958 char *servername = NULL;
966 @@ -500,6 +517,8 @@ int main(int argc, char *argv[])
968 int num_cq_events = 0;
973 srand48(getpid() * time(NULL));
975 @@ -515,10 +534,11 @@ int main(int argc, char *argv[])
976 { .name = "iters", .has_arg = 1, .val = 'n' },
977 { .name = "sl", .has_arg = 1, .val = 'l' },
978 { .name = "events", .has_arg = 0, .val = 'e' },
979 + { .name = "gid-idx", .has_arg = 1, .val = 'g' },
983 - c = getopt_long(argc, argv, "p:d:i:s:r:n:l:e", long_options, NULL);
984 + c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL);
988 @@ -563,6 +583,10 @@ int main(int argc, char *argv[])
993 + gidx = strtol(optarg, NULL, 0);
999 @@ -618,30 +642,41 @@ int main(int argc, char *argv[])
1003 - my_dest.lid = pp_get_local_lid(ctx->context, ib_port);
1004 - my_dest.qpn = ctx->qp->qp_num;
1005 - my_dest.psn = lrand48() & 0xffffff;
1006 - if (!my_dest.lid) {
1007 - fprintf(stderr, "Couldn't get local LID\n");
1008 + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
1009 + fprintf(stderr, "Couldn't get port info\n");
1012 + my_dest.lid = ctx->portinfo.lid;
1014 + my_dest.qpn = ctx->qp->qp_num;
1015 + my_dest.psn = lrand48() & 0xffffff;
1018 + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
1019 + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
1023 + memset(&my_dest.gid, 0, sizeof my_dest.gid);
1025 - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
1026 - my_dest.lid, my_dest.qpn, my_dest.psn);
1027 + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
1028 + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
1029 + my_dest.lid, my_dest.qpn, my_dest.psn, gid);
1032 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
1034 - rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest);
1035 + rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx);
1040 - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
1041 - rem_dest->lid, rem_dest->qpn, rem_dest->psn);
1042 + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
1043 + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
1044 + rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
1047 - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest))
1048 + if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx))
1051 ctx->pending = PINGPONG_RECV_WRID;