2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <sys/types.h>
40 #include <sys/socket.h>
45 #include <arpa/inet.h>
51 PINGPONG_RECV_WRID = 1,
52 PINGPONG_SEND_WRID = 2,
57 struct pingpong_context {
58 struct ibv_context *context;
59 struct ibv_comp_channel *channel;
70 struct ibv_port_attr portinfo;
73 struct pingpong_dest {
80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
81 int sl, struct pingpong_dest *dest, int sgid_idx)
83 struct ibv_ah_attr ah_attr = {
90 struct ibv_qp_attr attr = {
91 .qp_state = IBV_QPS_RTR
94 if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
95 fprintf(stderr, "Failed to modify QP to RTR\n");
99 attr.qp_state = IBV_QPS_RTS;
100 attr.sq_psn = my_psn;
102 if (ibv_modify_qp(ctx->qp, &attr,
105 fprintf(stderr, "Failed to modify QP to RTS\n");
109 if (dest->gid.global.interface_id) {
110 ah_attr.is_global = 1;
111 ah_attr.grh.hop_limit = 1;
112 ah_attr.grh.dgid = dest->gid;
113 ah_attr.grh.sgid_index = sgid_idx;
116 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
118 fprintf(stderr, "Failed to create AH\n");
125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
126 const struct pingpong_dest *my_dest)
128 struct addrinfo *res, *t;
129 struct addrinfo hints = {
130 .ai_family = AF_UNSPEC,
131 .ai_socktype = SOCK_STREAM
134 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
137 struct pingpong_dest *rem_dest = NULL;
140 if (asprintf(&service, "%d", port) < 0)
143 n = getaddrinfo(servername, service, &hints, &res);
146 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
151 for (t = res; t; t = t->ai_next) {
152 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
154 if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
161 freeaddrinfo_null(res);
165 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
169 gid_to_wire_gid(&my_dest->gid, gid);
170 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
172 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
173 fprintf(stderr, "Couldn't send local address\n");
177 if (read(sockfd, msg, sizeof msg) != sizeof msg ||
178 write(sockfd, "done", sizeof "done") != sizeof "done") {
179 perror("client read/write");
180 fprintf(stderr, "Couldn't read/write remote address\n");
184 rem_dest = malloc(sizeof *rem_dest);
188 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
189 &rem_dest->psn, gid);
190 wire_gid_to_gid(gid, &rem_dest->gid);
197 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
198 int ib_port, int port, int sl,
199 const struct pingpong_dest *my_dest,
202 struct addrinfo *res, *t;
203 struct addrinfo hints = {
204 .ai_flags = AI_PASSIVE,
205 .ai_family = AF_INET,
206 .ai_socktype = SOCK_STREAM
209 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
211 int sockfd = -1, connfd;
212 struct pingpong_dest *rem_dest = NULL;
215 if (asprintf(&service, "%d", port) < 0)
218 n = getaddrinfo(NULL, service, &hints, &res);
221 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
226 for (t = res; t; t = t->ai_next) {
227 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
231 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
233 if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
240 freeaddrinfo_null(res);
244 fprintf(stderr, "Couldn't listen to port %d\n", port);
249 connfd = accept(sockfd, NULL, NULL);
252 fprintf(stderr, "accept() failed\n");
256 n = read(connfd, msg, sizeof msg);
257 if (n != sizeof msg) {
258 perror("server read");
259 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
263 rem_dest = malloc(sizeof *rem_dest);
267 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
268 &rem_dest->psn, gid);
269 wire_gid_to_gid(gid, &rem_dest->gid);
271 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
273 fprintf(stderr, "Couldn't connect to remote QP\n");
279 gid_to_wire_gid(&my_dest->gid, gid);
280 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
282 if (write(connfd, msg, sizeof msg) != sizeof msg ||
283 read(connfd, msg, sizeof msg) != sizeof "done") {
284 fprintf(stderr, "Couldn't send/recv local address\n");
294 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
295 int rx_depth, int port,
298 struct pingpong_context *ctx;
300 ctx = malloc(sizeof *ctx);
305 ctx->send_flags = IBV_SEND_SIGNALED;
306 ctx->rx_depth = rx_depth;
308 ctx->buf = memalign(page_size, size + 40);
310 fprintf(stderr, "Couldn't allocate work buf.\n");
314 /* FIXME memset(ctx->buf, 0, size + 40); */
315 memset(ctx->buf, 0x7b, size + 40);
317 ctx->context = ibv_open_device(ib_dev);
319 fprintf(stderr, "Couldn't get context for %s\n",
320 ibv_get_device_name(ib_dev));
325 struct ibv_port_attr port_info = {};
328 if (ibv_query_port(ctx->context, port, &port_info)) {
329 fprintf(stderr, "Unable to query port info for port %d\n", port);
332 mtu = 1 << (port_info.active_mtu + 7);
334 fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
340 ctx->channel = ibv_create_comp_channel(ctx->context);
342 fprintf(stderr, "Couldn't create completion channel\n");
348 ctx->pd = ibv_alloc_pd(ctx->context);
350 fprintf(stderr, "Couldn't allocate PD\n");
351 goto clean_comp_channel;
354 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
356 fprintf(stderr, "Couldn't register MR\n");
360 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
363 fprintf(stderr, "Couldn't create CQ\n");
368 struct ibv_qp_attr attr;
369 struct ibv_qp_init_attr init_attr = {
374 .max_recv_wr = rx_depth,
378 .qp_type = IBV_QPT_UD,
381 ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
383 fprintf(stderr, "Couldn't create QP\n");
387 ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
388 if (init_attr.cap.max_inline_data >= size) {
389 ctx->send_flags |= IBV_SEND_INLINE;
394 struct ibv_qp_attr attr = {
395 .qp_state = IBV_QPS_INIT,
401 if (ibv_modify_qp(ctx->qp, &attr,
406 fprintf(stderr, "Failed to modify QP to INIT\n");
414 ibv_destroy_qp(ctx->qp);
417 ibv_destroy_cq(ctx->cq);
420 ibv_dereg_mr(ctx->mr);
423 ibv_dealloc_pd(ctx->pd);
427 ibv_destroy_comp_channel(ctx->channel);
430 ibv_close_device(ctx->context);
441 static int pp_close_ctx(struct pingpong_context *ctx)
443 if (ibv_destroy_qp(ctx->qp)) {
444 fprintf(stderr, "Couldn't destroy QP\n");
448 if (ibv_destroy_cq(ctx->cq)) {
449 fprintf(stderr, "Couldn't destroy CQ\n");
453 if (ibv_dereg_mr(ctx->mr)) {
454 fprintf(stderr, "Couldn't deregister MR\n");
458 if (ibv_destroy_ah(ctx->ah)) {
459 fprintf(stderr, "Couldn't destroy AH\n");
463 if (ibv_dealloc_pd(ctx->pd)) {
464 fprintf(stderr, "Couldn't deallocate PD\n");
469 if (ibv_destroy_comp_channel(ctx->channel)) {
470 fprintf(stderr, "Couldn't destroy completion channel\n");
475 if (ibv_close_device(ctx->context)) {
476 fprintf(stderr, "Couldn't release context\n");
486 static int pp_post_recv(struct pingpong_context *ctx, int n)
488 struct ibv_sge list = {
489 .addr = (uintptr_t) ctx->buf,
490 .length = ctx->size + 40,
491 .lkey = ctx->mr->lkey
493 struct ibv_recv_wr wr = {
494 .wr_id = PINGPONG_RECV_WRID,
498 struct ibv_recv_wr *bad_wr;
501 for (i = 0; i < n; ++i)
502 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
508 static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
510 struct ibv_sge list = {
511 .addr = (uintptr_t) ctx->buf + 40,
513 .lkey = ctx->mr->lkey
515 struct ibv_send_wr wr = {
516 .wr_id = PINGPONG_SEND_WRID,
519 .opcode = IBV_WR_SEND,
520 .send_flags = ctx->send_flags,
525 .remote_qkey = 0x11111111
529 struct ibv_send_wr *bad_wr;
531 return ibv_post_send(ctx->qp, &wr, &bad_wr);
534 static void usage(const char *argv0)
537 printf(" %s start a server and wait for connection\n", argv0);
538 printf(" %s <host> connect to server at <host>\n", argv0);
540 printf("Options:\n");
541 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
542 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
543 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
544 printf(" -s, --size=<size> size of message to exchange (default 2048)\n");
545 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
546 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
547 printf(" -l, --sl=<SL> send messages with service level <SL> (default 0)\n");
548 printf(" -e, --events sleep on CQ events (default poll)\n");
549 printf(" -g, --gid-idx=<gid index> local port gid index\n");
552 int main(int argc, char *argv[])
554 struct ibv_device **dev_list;
555 struct ibv_device *ib_dev;
556 struct pingpong_context *ctx;
557 struct pingpong_dest my_dest;
558 struct pingpong_dest *rem_dest;
559 struct timeval start, end;
560 char *ib_devname = NULL;
561 char *servername = NULL;
562 unsigned int port = 18515;
564 unsigned int size = 2048;
565 unsigned int rx_depth = 500;
566 unsigned int iters = 1000;
570 int num_cq_events = 0;
575 srand48(getpid() * time(NULL));
580 static struct option long_options[] = {
581 { .name = "port", .has_arg = 1, .val = 'p' },
582 { .name = "ib-dev", .has_arg = 1, .val = 'd' },
583 { .name = "ib-port", .has_arg = 1, .val = 'i' },
584 { .name = "size", .has_arg = 1, .val = 's' },
585 { .name = "rx-depth", .has_arg = 1, .val = 'r' },
586 { .name = "iters", .has_arg = 1, .val = 'n' },
587 { .name = "sl", .has_arg = 1, .val = 'l' },
588 { .name = "events", .has_arg = 0, .val = 'e' },
589 { .name = "gid-idx", .has_arg = 1, .val = 'g' },
593 c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
600 port = strtol(optarg, NULL, 0);
608 ib_devname = strdupa(optarg);
612 ib_port = strtol(optarg, NULL, 0);
620 size = strtoul(optarg, NULL, 0);
624 rx_depth = strtoul(optarg, NULL, 0);
628 iters = strtoul(optarg, NULL, 0);
632 sl = strtol(optarg, NULL, 0);
640 gidx = strtol(optarg, NULL, 0);
649 if (optind == argc - 1)
650 servername = strdupa(argv[optind]);
651 else if (optind < argc) {
656 page_size = sysconf(_SC_PAGESIZE);
658 dev_list = ibv_get_device_list(NULL);
660 perror("Failed to get IB devices list");
667 fprintf(stderr, "No IB devices found\n");
672 for (i = 0; dev_list[i]; ++i)
673 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
675 ib_dev = dev_list[i];
677 fprintf(stderr, "IB device %s not found\n", ib_devname);
682 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
686 routs = pp_post_recv(ctx, ctx->rx_depth);
687 if (routs < ctx->rx_depth) {
688 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
693 if (ibv_req_notify_cq(ctx->cq, 0)) {
694 fprintf(stderr, "Couldn't request CQ notification\n");
698 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
699 fprintf(stderr, "Couldn't get port info\n");
702 my_dest.lid = ctx->portinfo.lid;
704 my_dest.qpn = ctx->qp->qp_num;
705 my_dest.psn = lrand48() & 0xffffff;
708 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
709 fprintf(stderr, "Could not get local gid for gid index "
714 memset(&my_dest.gid, 0, sizeof my_dest.gid);
716 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
717 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
718 my_dest.lid, my_dest.qpn, my_dest.psn, gid);
721 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
723 rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl,
729 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
730 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
731 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
734 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest,
738 ctx->pending = PINGPONG_RECV_WRID;
741 if (pp_post_send(ctx, rem_dest->qpn)) {
742 fprintf(stderr, "Couldn't post send\n");
745 ctx->pending |= PINGPONG_SEND_WRID;
748 if (gettimeofday(&start, NULL)) {
749 perror("gettimeofday");
754 while (rcnt < iters || scnt < iters) {
756 struct ibv_cq *ev_cq;
759 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
760 fprintf(stderr, "Failed to get cq_event\n");
766 if (ev_cq != ctx->cq) {
767 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
771 if (ibv_req_notify_cq(ctx->cq, 0)) {
772 fprintf(stderr, "Couldn't request CQ notification\n");
782 ne = ibv_poll_cq(ctx->cq, 2, wc);
784 fprintf(stderr, "poll CQ failed %d\n", ne);
787 } while (!use_event && ne < 1);
789 for (i = 0; i < ne; ++i) {
790 if (wc[i].status != IBV_WC_SUCCESS) {
791 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
792 ibv_wc_status_str(wc[i].status),
793 wc[i].status, (int) wc[i].wr_id);
797 switch ((int) wc[i].wr_id) {
798 case PINGPONG_SEND_WRID:
802 case PINGPONG_RECV_WRID:
804 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
805 if (routs < ctx->rx_depth) {
807 "Couldn't post receive (%d)\n",
817 fprintf(stderr, "Completion for unknown wr_id %d\n",
822 ctx->pending &= ~(int) wc[i].wr_id;
823 if (scnt < iters && !ctx->pending) {
824 if (pp_post_send(ctx, rem_dest->qpn)) {
825 fprintf(stderr, "Couldn't post send\n");
828 ctx->pending = PINGPONG_RECV_WRID |
835 if (gettimeofday(&end, NULL)) {
836 perror("gettimeofday");
841 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
842 (end.tv_usec - start.tv_usec);
843 long long bytes = (long long) size * iters * 2;
845 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
846 bytes, usec / 1000000., bytes * 8. / usec);
847 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
848 iters, usec / 1000000., usec / iters);
851 ibv_ack_cq_events(ctx->cq, num_cq_events);
853 if (pp_close_ctx(ctx))
856 ibv_free_device_list(dev_list);