contrib/ofed/libibverbs/examples/uc_pingpong.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #if HAVE_CONFIG_H
  34 #  include <config.h>
  35 #endif /* HAVE_CONFIG_H */
  36
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <unistd.h>
  40 #include <string.h>
  41 #include <sys/types.h>
  42 #include <sys/socket.h>
  43 #include <sys/time.h>
  44 #include <netdb.h>
  45 #include <getopt.h>
  46 #include <arpa/inet.h>
  47 #include <time.h>
  48
  49 #include "pingpong.h"
  50
  51 enum {
  52         PINGPONG_RECV_WRID = 1,
  53         PINGPONG_SEND_WRID = 2,
  54 };
  55
  56 static int page_size;
  57
  58 struct pingpong_context {
  59         struct ibv_context      *context;
  60         struct ibv_comp_channel *channel;
  61         struct ibv_pd           *pd;
  62         struct ibv_mr           *mr;
  63         struct ibv_cq           *cq;
  64         struct ibv_qp           *qp;
  65         void                    *buf;
  66         int                      size;
  67         int                      rx_depth;
  68         int                      pending;
  69         struct ibv_port_attr     portinfo;
  70 };
  71
  72 struct pingpong_dest {
  73         int lid;
  74         int qpn;
  75         int psn;
  76         union ibv_gid gid;
  77 };
  78
  79 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
  80                           enum ibv_mtu mtu, int sl,
  81                           struct pingpong_dest *dest, int sgid_idx)
  82 {
  83         struct ibv_qp_attr attr = {
  84                 .qp_state               = IBV_QPS_RTR,
  85                 .path_mtu               = mtu,
  86                 .dest_qp_num            = dest->qpn,
  87                 .rq_psn         = dest->psn,
  88                 .ah_attr                = {
  89                         .is_global      = 0,
  90                         .dlid           = dest->lid,
  91                         .sl             = sl,
  92                         .src_path_bits  = 0,
  93                         .port_num       = port
  94                 }
  95         };
  96
  97         if (dest->gid.global.interface_id) {
  98                 attr.ah_attr.is_global = 1;
  99                 attr.ah_attr.grh.hop_limit = 1;
 100                 attr.ah_attr.grh.dgid = dest->gid;
 101                 attr.ah_attr.grh.sgid_index = sgid_idx;
 102         }
 103
 104         if (ibv_modify_qp(ctx->qp, &attr,
 105                           IBV_QP_STATE              |
 106                           IBV_QP_AV                 |
 107                           IBV_QP_PATH_MTU           |
 108                           IBV_QP_DEST_QPN           |
 109                           IBV_QP_RQ_PSN)) {
 110                 fprintf(stderr, "Failed to modify QP to RTR\n");
 111                 return 1;
 112         }
 113
 114         attr.qp_state       = IBV_QPS_RTS;
 115         attr.sq_psn         = my_psn;
 116         if (ibv_modify_qp(ctx->qp, &attr,
 117                           IBV_QP_STATE              |
 118                           IBV_QP_SQ_PSN)) {
 119                 fprintf(stderr, "Failed to modify QP to RTS\n");
 120                 return 1;
 121         }
 122
 123         return 0;
 124 }
 125
 126 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 127                                                  const struct pingpong_dest *my_dest)
 128 {
 129         struct addrinfo *res, *t;
 130         struct addrinfo hints = {
 131                 .ai_family   = AF_INET,
 132                 .ai_socktype = SOCK_STREAM
 133         };
 134         char *service;
 135         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 136         int n;
 137         int sockfd = -1;
 138         struct pingpong_dest *rem_dest = NULL;
 139         char gid[33];
 140
 141         if (asprintf(&service, "%d", port) < 0)
 142                 return NULL;
 143
 144         n = getaddrinfo(servername, service, &hints, &res);
 145
 146         if (n < 0) {
 147                 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
 148                 free(service);
 149                 return NULL;
 150         }
 151
 152         for (t = res; t; t = t->ai_next) {
 153                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 154                 if (sockfd >= 0) {
 155                         if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
 156                                 break;
 157                         close(sockfd);
 158                         sockfd = -1;
 159                 }
 160         }
 161
 162         freeaddrinfo(res);
 163         free(service);
 164
 165         if (sockfd < 0) {
 166                 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
 167                 return NULL;
 168         }
 169
 170         gid_to_wire_gid(&my_dest->gid, gid);
 171         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
 172         if (write(sockfd, msg, sizeof msg) != sizeof msg) {
 173                 fprintf(stderr, "Couldn't send local address\n");
 174                 goto out;
 175         }
 176
 177         if (read(sockfd, msg, sizeof msg) != sizeof msg) {
 178                 perror("client read");
 179                 fprintf(stderr, "Couldn't read remote address\n");
 180                 goto out;
 181         }
 182
 183         write(sockfd, "done", sizeof "done");
 184
 185         rem_dest = malloc(sizeof *rem_dest);
 186         if (!rem_dest)
 187                 goto out;
 188
 189         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
 190         wire_gid_to_gid(gid, &rem_dest->gid);
 191
 192 out:
 193         close(sockfd);
 194         return rem_dest;
 195 }
 196
 197 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
 198                                                  int ib_port, enum ibv_mtu mtu,
 199                                                  int port, int sl,
 200                                                  const struct pingpong_dest *my_dest,
 201                                                  int sgid_idx)
 202 {
 203         struct addrinfo *res, *t;
 204         struct addrinfo hints = {
 205                 .ai_flags    = AI_PASSIVE,
 206                 .ai_family   = AF_INET,
 207                 .ai_socktype = SOCK_STREAM
 208         };
 209         char *service;
 210         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 211         int n;
 212         int sockfd = -1, connfd;
 213         struct pingpong_dest *rem_dest = NULL;
 214         char gid[33];
 215
 216         if (asprintf(&service, "%d", port) < 0)
 217                 return NULL;
 218
 219         n = getaddrinfo(NULL, service, &hints, &res);
 220
 221         if (n < 0) {
 222                 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
 223                 free(service);
 224                 return NULL;
 225         }
 226
 227         for (t = res; t; t = t->ai_next) {
 228                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 229                 if (sockfd >= 0) {
 230                         n = 1;
 231
 232                         setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
 233
 234                         if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
 235                                 break;
 236                         close(sockfd);
 237                         sockfd = -1;
 238                 }
 239         }
 240
 241         freeaddrinfo(res);
 242         free(service);
 243
 244         if (sockfd < 0) {
 245                 fprintf(stderr, "Couldn't listen to port %d\n", port);
 246                 return NULL;
 247         }
 248
 249         listen(sockfd, 1);
 250         connfd = accept(sockfd, NULL, 0);
 251         close(sockfd);
 252         if (connfd < 0) {
 253                 fprintf(stderr, "accept() failed\n");
 254                 return NULL;
 255         }
 256
 257         n = read(connfd, msg, sizeof msg);
 258         if (n != sizeof msg) {
 259                 perror("server read");
 260                 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
 261                 goto out;
 262         }
 263
 264         rem_dest = malloc(sizeof *rem_dest);
 265         if (!rem_dest)
 266                 goto out;
 267
 268         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
 269         wire_gid_to_gid(gid, &rem_dest->gid);
 270
 271         if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) {
 272                 fprintf(stderr, "Couldn't connect to remote QP\n");
 273                 free(rem_dest);
 274                 rem_dest = NULL;
 275                 goto out;
 276         }
 277
 278         gid_to_wire_gid(&my_dest->gid, gid);
 279         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
 280         if (write(connfd, msg, sizeof msg) != sizeof msg) {
 281                 fprintf(stderr, "Couldn't send local address\n");
 282                 free(rem_dest);
 283                 rem_dest = NULL;
 284                 goto out;
 285         }
 286
 287         read(connfd, msg, sizeof msg);
 288
 289 out:
 290         close(connfd);
 291         return rem_dest;
 292 }
 293
 294 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 295                                             int rx_depth, int port,
 296                                             int use_event)
 297 {
 298         struct pingpong_context *ctx;
 299
 300         ctx = calloc(1, sizeof *ctx);
 301         if (!ctx)
 302                 return NULL;
 303
 304         ctx->size     = size;
 305         ctx->rx_depth = rx_depth;
 306
 307         ctx->buf = malloc(roundup(size, page_size));
 308         if (!ctx->buf) {
 309                 fprintf(stderr, "Couldn't allocate work buf.\n");
 310                 return NULL;
 311         }
 312
 313         memset(ctx->buf, 0, size);
 314
 315         ctx->context = ibv_open_device(ib_dev);
 316         if (!ctx->context) {
 317                 fprintf(stderr, "Couldn't get context for %s\n",
 318                         ibv_get_device_name(ib_dev));
 319                 return NULL;
 320         }
 321
 322         if (use_event) {
 323                 ctx->channel = ibv_create_comp_channel(ctx->context);
 324                 if (!ctx->channel) {
 325                         fprintf(stderr, "Couldn't create completion channel\n");
 326                         return NULL;
 327                 }
 328         } else
 329                 ctx->channel = NULL;
 330
 331         ctx->pd = ibv_alloc_pd(ctx->context);
 332         if (!ctx->pd) {
 333                 fprintf(stderr, "Couldn't allocate PD\n");
 334                 return NULL;
 335         }
 336
 337         ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE);
 338         if (!ctx->mr) {
 339                 fprintf(stderr, "Couldn't register MR\n");
 340                 return NULL;
 341         }
 342
 343         ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
 344                                 ctx->channel, 0);
 345         if (!ctx->cq) {
 346                 fprintf(stderr, "Couldn't create CQ\n");
 347                 return NULL;
 348         }
 349
 350         {
 351                 struct ibv_qp_init_attr attr = {
 352                         .send_cq = ctx->cq,
 353                         .recv_cq = ctx->cq,
 354                         .cap     = {
 355                                 .max_send_wr  = 1,
 356                                 .max_recv_wr  = rx_depth,
 357                                 .max_send_sge = 1,
 358                                 .max_recv_sge = 1
 359                         },
 360                         .qp_type = IBV_QPT_UC
 361                 };
 362
 363                 ctx->qp = ibv_create_qp(ctx->pd, &attr);
 364                 if (!ctx->qp)  {
 365                         fprintf(stderr, "Couldn't create QP\n");
 366                         return NULL;
 367                 }
 368         }
 369
 370         {
 371                 struct ibv_qp_attr attr = {
 372                         .qp_state        = IBV_QPS_INIT,
 373                         .pkey_index      = 0,
 374                         .port_num        = port,
 375                         .qp_access_flags = 0
 376                 };
 377
 378                 if (ibv_modify_qp(ctx->qp, &attr,
 379                                   IBV_QP_STATE              |
 380                                   IBV_QP_PKEY_INDEX         |
 381                                   IBV_QP_PORT               |
 382                                   IBV_QP_ACCESS_FLAGS)) {
 383                         fprintf(stderr, "Failed to modify QP to INIT\n");
 384                         return NULL;
 385                 }
 386         }
 387
 388         return ctx;
 389 }
 390
 391 int pp_close_ctx(struct pingpong_context *ctx)
 392 {
 393         if (ibv_destroy_qp(ctx->qp)) {
 394                 fprintf(stderr, "Couldn't destroy QP\n");
 395                 return 1;
 396         }
 397
 398         if (ibv_destroy_cq(ctx->cq)) {
 399                 fprintf(stderr, "Couldn't destroy CQ\n");
 400                 return 1;
 401         }
 402
 403         if (ibv_dereg_mr(ctx->mr)) {
 404                 fprintf(stderr, "Couldn't deregister MR\n");
 405                 return 1;
 406         }
 407
 408         if (ibv_dealloc_pd(ctx->pd)) {
 409                 fprintf(stderr, "Couldn't deallocate PD\n");
 410                 return 1;
 411         }
 412
 413         if (ctx->channel) {
 414                 if (ibv_destroy_comp_channel(ctx->channel)) {
 415                         fprintf(stderr, "Couldn't destroy completion channel\n");
 416                         return 1;
 417                 }
 418         }
 419
 420         if (ibv_close_device(ctx->context)) {
 421                 fprintf(stderr, "Couldn't release context\n");
 422                 return 1;
 423         }
 424
 425         free(ctx->buf);
 426         free(ctx);
 427
 428         return 0;
 429 }
 430
 431 static int pp_post_recv(struct pingpong_context *ctx, int n)
 432 {
 433         struct ibv_sge list = {
 434                 .addr   = (uintptr_t) ctx->buf,
 435                 .length = ctx->size,
 436                 .lkey   = ctx->mr->lkey
 437         };
 438         struct ibv_recv_wr wr = {
 439                 .wr_id      = PINGPONG_RECV_WRID,
 440                 .sg_list    = &list,
 441                 .num_sge    = 1,
 442         };
 443         struct ibv_recv_wr *bad_wr;
 444         int i;
 445
 446         for (i = 0; i < n; ++i)
 447                 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
 448                         break;
 449
 450         return i;
 451 }
 452
 453 static int pp_post_send(struct pingpong_context *ctx)
 454 {
 455         struct ibv_sge list = {
 456                 .addr   = (uintptr_t) ctx->buf,
 457                 .length = ctx->size,
 458                 .lkey   = ctx->mr->lkey
 459         };
 460         struct ibv_send_wr wr = {
 461                 .wr_id      = PINGPONG_SEND_WRID,
 462                 .sg_list    = &list,
 463                 .num_sge    = 1,
 464                 .opcode     = IBV_WR_SEND,
 465                 .send_flags = IBV_SEND_SIGNALED,
 466         };
 467         struct ibv_send_wr *bad_wr;
 468
 469         return ibv_post_send(ctx->qp, &wr, &bad_wr);
 470 }
 471
 472 static void usage(const char *argv0)
 473 {
 474         printf("Usage:\n");
 475         printf("  %s            start a server and wait for connection\n", argv0);
 476         printf("  %s <host>     connect to server at <host>\n", argv0);
 477         printf("\n");
 478         printf("Options:\n");
 479         printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
 480         printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
 481         printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
 482         printf("  -s, --size=<size>      size of message to exchange (default 4096)\n");
 483         printf("  -m, --mtu=<size>       path MTU (default 1024)\n");
 484         printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
 485         printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
 486         printf("  -l, --sl=<sl>          service level value\n");
 487         printf("  -e, --events           sleep on CQ events (default poll)\n");
 488         printf("  -g, --gid-idx=<gid index> local port gid index\n");
 489 }
 490
 491 int main(int argc, char *argv[])
 492 {
 493         struct ibv_device      **dev_list;
 494         struct ibv_device       *ib_dev;
 495         struct pingpong_context *ctx;
 496         struct pingpong_dest     my_dest;
 497         struct pingpong_dest    *rem_dest;
 498         struct timeval           start, end;
 499         char                    *ib_devname = NULL;
 500         char                    *servername = NULL;
 501         int                      port = 18515;
 502         int                      ib_port = 1;
 503         int                      size = 4096;
 504         enum ibv_mtu             mtu = IBV_MTU_1024;
 505         int                      rx_depth = 500;
 506         int                      iters = 1000;
 507         int                      use_event = 0;
 508         int                      routs;
 509         int                      rcnt, scnt;
 510         int                      num_cq_events = 0;
 511         int                      sl = 0;
 512         int                      gidx = -1;
 513         char                     gid[33];
 514
 515         srand48(getpid() * time(NULL));
 516
 517         while (1) {
 518                 int c;
 519
 520                 static struct option long_options[] = {
 521                         { .name = "port",     .has_arg = 1, .val = 'p' },
 522                         { .name = "ib-dev",   .has_arg = 1, .val = 'd' },
 523                         { .name = "ib-port",  .has_arg = 1, .val = 'i' },
 524                         { .name = "size",     .has_arg = 1, .val = 's' },
 525                         { .name = "mtu",      .has_arg = 1, .val = 'm' },
 526                         { .name = "rx-depth", .has_arg = 1, .val = 'r' },
 527                         { .name = "iters",    .has_arg = 1, .val = 'n' },
 528                         { .name = "sl",       .has_arg = 1, .val = 'l' },
 529                         { .name = "events",   .has_arg = 0, .val = 'e' },
 530                         { .name = "gid-idx",  .has_arg = 1, .val = 'g' },
 531                         { 0 }
 532                 };
 533
 534                 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL);
 535                 if (c == -1)
 536                         break;
 537
 538                 switch (c) {
 539                 case 'p':
 540                         port = strtol(optarg, NULL, 0);
 541                         if (port < 0 || port > 65535) {
 542                                 usage(argv[0]);
 543                                 return 1;
 544                         }
 545                         break;
 546
 547                 case 'd':
 548                         ib_devname = strdup(optarg);
 549                         break;
 550
 551                 case 'i':
 552                         ib_port = strtol(optarg, NULL, 0);
 553                         if (ib_port < 0) {
 554                                 usage(argv[0]);
 555                                 return 1;
 556                         }
 557                         break;
 558
 559                 case 's':
 560                         size = strtol(optarg, NULL, 0);
 561                         break;
 562
 563                 case 'm':
 564                         mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
 565                         if (mtu < 0) {
 566                                 usage(argv[0]);
 567                                 return 1;
 568                         }
 569                         break;
 570
 571                 case 'r':
 572                         rx_depth = strtol(optarg, NULL, 0);
 573                         break;
 574
 575                 case 'n':
 576                         iters = strtol(optarg, NULL, 0);
 577                         break;
 578
 579                 case 'l':
 580                         sl = strtol(optarg, NULL, 0);
 581                         break;
 582
 583                 case 'e':
 584                         ++use_event;
 585                         break;
 586
 587                 case 'g':
 588                         gidx = strtol(optarg, NULL, 0);
 589                         break;
 590
 591                 default:
 592                         usage(argv[0]);
 593                         return 1;
 594                 }
 595         }
 596
 597         if (optind == argc - 1)
 598                 servername = strdup(argv[optind]);
 599         else if (optind < argc) {
 600                 usage(argv[0]);
 601                 return 1;
 602         }
 603
 604         page_size = sysconf(_SC_PAGESIZE);
 605
 606         dev_list = ibv_get_device_list(NULL);
 607         if (!dev_list) {
 608                 perror("Failed to get IB devices list");
 609                 return 1;
 610         }
 611
 612         if (!ib_devname) {
 613                 ib_dev = *dev_list;
 614                 if (!ib_dev) {
 615                         fprintf(stderr, "No IB devices found\n");
 616                         return 1;
 617                 }
 618         } else {
 619                 int i;
 620                 for (i = 0; dev_list[i]; ++i)
 621                         if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
 622                                 break;
 623                 ib_dev = dev_list[i];
 624                 if (!ib_dev) {
 625                         fprintf(stderr, "IB device %s not found\n", ib_devname);
 626                         return 1;
 627                 }
 628         }
 629
 630         ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
 631         if (!ctx)
 632                 return 1;
 633
 634         routs = pp_post_recv(ctx, ctx->rx_depth);
 635         if (routs < ctx->rx_depth) {
 636                 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
 637                 return 1;
 638         }
 639
 640         if (use_event)
 641                 if (ibv_req_notify_cq(ctx->cq, 0)) {
 642                         fprintf(stderr, "Couldn't request CQ notification\n");
 643                         return 1;
 644                 }
 645
 646         if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
 647                 fprintf(stderr, "Couldn't get port info\n");
 648                 return 1;
 649         }
 650
 651         my_dest.lid = ctx->portinfo.lid;
 652         if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) {
 653                 fprintf(stderr, "Couldn't get local LID\n");
 654                 return 1;
 655         }
 656
 657         if (gidx >= 0) {
 658                 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
 659                         fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
 660                         return 1;
 661                 }
 662         } else
 663                 memset(&my_dest.gid, 0, sizeof my_dest.gid);
 664
 665         my_dest.qpn = ctx->qp->qp_num;
 666         my_dest.psn = lrand48() & 0xffffff;
 667         inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
 668         printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 669                my_dest.lid, my_dest.qpn, my_dest.psn, gid);
 670
 671         if (servername)
 672                 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 673         else
 674                 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
 675
 676         if (!rem_dest)
 677                 return 1;
 678
 679         inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
 680         printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 681                rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
 682
 683         if (servername)
 684                 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx))
 685                         return 1;
 686
 687         ctx->pending = PINGPONG_RECV_WRID;
 688
 689         if (servername) {
 690                 if (pp_post_send(ctx)) {
 691                         fprintf(stderr, "Couldn't post send\n");
 692                         return 1;
 693                 }
 694                 ctx->pending |= PINGPONG_SEND_WRID;
 695         }
 696
 697         if (gettimeofday(&start, NULL)) {
 698                 perror("gettimeofday");
 699                 return 1;
 700         }
 701
 702         rcnt = scnt = 0;
 703         while (rcnt < iters || scnt < iters) {
 704                 if (use_event) {
 705                         struct ibv_cq *ev_cq;
 706                         void          *ev_ctx;
 707
 708                         if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
 709                                 fprintf(stderr, "Failed to get cq_event\n");
 710                                 return 1;
 711                         }
 712
 713                         ++num_cq_events;
 714
 715                         if (ev_cq != ctx->cq) {
 716                                 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
 717                                 return 1;
 718                         }
 719
 720                         if (ibv_req_notify_cq(ctx->cq, 0)) {
 721                                 fprintf(stderr, "Couldn't request CQ notification\n");
 722                                 return 1;
 723                         }
 724                 }
 725
 726                 {
 727                         struct ibv_wc wc[2];
 728                         int ne, i;
 729
 730                         do {
 731                                 ne = ibv_poll_cq(ctx->cq, 2, wc);
 732                                 if (ne < 0) {
 733                                         fprintf(stderr, "poll CQ failed %d\n", ne);
 734                                         return 1;
 735                                 }
 736                         } while (!use_event && ne < 1);
 737
 738                         for (i = 0; i < ne; ++i) {
 739                                 if (wc[i].status != IBV_WC_SUCCESS) {
 740                                         fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
 741                                                 ibv_wc_status_str(wc[i].status),
 742                                                 wc[i].status, (int) wc[i].wr_id);
 743                                         return 1;
 744                                 }
 745
 746                                 switch ((int) wc[i].wr_id) {
 747                                 case PINGPONG_SEND_WRID:
 748                                         ++scnt;
 749                                         break;
 750
 751                                 case PINGPONG_RECV_WRID:
 752                                         if (--routs <= 1) {
 753                                                 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
 754                                                 if (routs < ctx->rx_depth) {
 755                                                         fprintf(stderr,
 756                                                                 "Couldn't post receive (%d)\n",
 757                                                                 routs);
 758                                                         return 1;
 759                                                 }
 760                                         }
 761
 762                                         ++rcnt;
 763                                         break;
 764
 765                                 default:
 766                                         fprintf(stderr, "Completion for unknown wr_id %d\n",
 767                                                 (int) wc[i].wr_id);
 768                                         return 1;
 769                                 }
 770
 771                                 ctx->pending &= ~(int) wc[i].wr_id;
 772                                 if (scnt < iters && !ctx->pending) {
 773                                         if (pp_post_send(ctx)) {
 774                                                 fprintf(stderr, "Couldn't post send\n");
 775                                                 return 1;
 776                                         }
 777                                         ctx->pending = PINGPONG_RECV_WRID |
 778                                                        PINGPONG_SEND_WRID;
 779                                 }
 780                         }
 781                 }
 782         }
 783
 784         if (gettimeofday(&end, NULL)) {
 785                 perror("gettimeofday");
 786                 return 1;
 787         }
 788
 789         {
 790                 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
 791                         (end.tv_usec - start.tv_usec);
 792                 long long bytes = (long long) size * iters * 2;
 793
 794                 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
 795                        bytes, usec / 1000000., bytes * 8. / usec);
 796                 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
 797                        iters, usec / 1000000., usec / iters);
 798         }
 799
 800         ibv_ack_cq_events(ctx->cq, num_cq_events);
 801
 802         if (pp_close_ctx(ctx))
 803                 return 1;
 804
 805         ibv_free_device_list(dev_list);
 806         free(rem_dest);
 807
 808         return 0;
 809 }