contrib/ofed/libibverbs/examples/rc_pingpong.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #if HAVE_CONFIG_H
  34 #  include <config.h>
  35 #endif /* HAVE_CONFIG_H */
  36
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <unistd.h>
  40 #include <string.h>
  41 #include <sys/types.h>
  42 #include <sys/socket.h>
  43 #include <sys/time.h>
  44 #include <netdb.h>
  45 #include <stdlib.h>
  46 #include <getopt.h>
  47 #include <arpa/inet.h>
  48 #include <time.h>
  49
  50 #include "pingpong.h"
  51
  52 enum {
  53         PINGPONG_RECV_WRID = 1,
  54         PINGPONG_SEND_WRID = 2,
  55 };
  56
  57 static int page_size;
  58
  59 struct pingpong_context {
  60         struct ibv_context      *context;
  61         struct ibv_comp_channel *channel;
  62         struct ibv_pd           *pd;
  63         struct ibv_mr           *mr;
  64         struct ibv_cq           *cq;
  65         struct ibv_qp           *qp;
  66         void                    *buf;
  67         int                      size;
  68         int                      rx_depth;
  69         int                      pending;
  70         struct ibv_port_attr     portinfo;
  71 };
  72
  73 struct pingpong_dest {
  74         int lid;
  75         int qpn;
  76         int psn;
  77         union ibv_gid gid;
  78 };
  79
  80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
  81                           enum ibv_mtu mtu, int sl,
  82                           struct pingpong_dest *dest, int sgid_idx)
  83 {
  84         struct ibv_qp_attr attr = {
  85                 .qp_state               = IBV_QPS_RTR,
  86                 .path_mtu               = mtu,
  87                 .dest_qp_num            = dest->qpn,
  88                 .rq_psn                 = dest->psn,
  89                 .max_dest_rd_atomic     = 1,
  90                 .min_rnr_timer          = 12,
  91                 .ah_attr                = {
  92                         .is_global      = 0,
  93                         .dlid           = dest->lid,
  94                         .sl             = sl,
  95                         .src_path_bits  = 0,
  96                         .port_num       = port
  97                 }
  98         };
  99
 100         if (dest->gid.global.interface_id) {
 101                 attr.ah_attr.is_global = 1;
 102                 attr.ah_attr.grh.hop_limit = 1;
 103                 attr.ah_attr.grh.dgid = dest->gid;
 104                 attr.ah_attr.grh.sgid_index = sgid_idx;
 105         }
 106         if (ibv_modify_qp(ctx->qp, &attr,
 107                           IBV_QP_STATE              |
 108                           IBV_QP_AV                 |
 109                           IBV_QP_PATH_MTU           |
 110                           IBV_QP_DEST_QPN           |
 111                           IBV_QP_RQ_PSN             |
 112                           IBV_QP_MAX_DEST_RD_ATOMIC |
 113                           IBV_QP_MIN_RNR_TIMER)) {
 114                 fprintf(stderr, "Failed to modify QP to RTR\n");
 115                 return 1;
 116         }
 117
 118         attr.qp_state       = IBV_QPS_RTS;
 119         attr.timeout        = 14;
 120         attr.retry_cnt      = 7;
 121         attr.rnr_retry      = 7;
 122         attr.sq_psn         = my_psn;
 123         attr.max_rd_atomic  = 1;
 124         if (ibv_modify_qp(ctx->qp, &attr,
 125                           IBV_QP_STATE              |
 126                           IBV_QP_TIMEOUT            |
 127                           IBV_QP_RETRY_CNT          |
 128                           IBV_QP_RNR_RETRY          |
 129                           IBV_QP_SQ_PSN             |
 130                           IBV_QP_MAX_QP_RD_ATOMIC)) {
 131                 fprintf(stderr, "Failed to modify QP to RTS\n");
 132                 return 1;
 133         }
 134
 135         return 0;
 136 }
 137
 138 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 139                                                  const struct pingpong_dest *my_dest)
 140 {
 141         struct addrinfo *res, *t;
 142         struct addrinfo hints = {
 143                 .ai_family   = AF_INET,
 144                 .ai_socktype = SOCK_STREAM
 145         };
 146         char *service;
 147         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 148         int n;
 149         int sockfd = -1;
 150         struct pingpong_dest *rem_dest = NULL;
 151         char gid[33];
 152
 153         if (asprintf(&service, "%d", port) < 0)
 154                 return NULL;
 155
 156         n = getaddrinfo(servername, service, &hints, &res);
 157
 158         if (n < 0) {
 159                 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
 160                 free(service);
 161                 return NULL;
 162         }
 163
 164         for (t = res; t; t = t->ai_next) {
 165                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 166                 if (sockfd >= 0) {
 167                         if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
 168                                 break;
 169                         close(sockfd);
 170                         sockfd = -1;
 171                 }
 172         }
 173
 174         freeaddrinfo(res);
 175         free(service);
 176
 177         if (sockfd < 0) {
 178                 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
 179                 return NULL;
 180         }
 181
 182         gid_to_wire_gid(&my_dest->gid, gid);
 183         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
 184         if (write(sockfd, msg, sizeof msg) != sizeof msg) {
 185                 fprintf(stderr, "Couldn't send local address\n");
 186                 goto out;
 187         }
 188
 189         if (read(sockfd, msg, sizeof msg) != sizeof msg) {
 190                 perror("client read");
 191                 fprintf(stderr, "Couldn't read remote address\n");
 192                 goto out;
 193         }
 194
 195         write(sockfd, "done", sizeof "done");
 196
 197         rem_dest = malloc(sizeof *rem_dest);
 198         if (!rem_dest)
 199                 goto out;
 200
 201         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
 202         wire_gid_to_gid(gid, &rem_dest->gid);
 203
 204 out:
 205         close(sockfd);
 206         return rem_dest;
 207 }
 208
 209 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
 210                                                  int ib_port, enum ibv_mtu mtu,
 211                                                  int port, int sl,
 212                                                  const struct pingpong_dest *my_dest,
 213                                                  int sgid_idx)
 214 {
 215         struct addrinfo *res, *t;
 216         struct addrinfo hints = {
 217                 .ai_flags    = AI_PASSIVE,
 218                 .ai_family   = AF_INET,
 219                 .ai_socktype = SOCK_STREAM
 220         };
 221         char *service;
 222         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 223         int n;
 224         int sockfd = -1, connfd;
 225         struct pingpong_dest *rem_dest = NULL;
 226         char gid[33];
 227
 228         if (asprintf(&service, "%d", port) < 0)
 229                 return NULL;
 230
 231         n = getaddrinfo(NULL, service, &hints, &res);
 232
 233         if (n < 0) {
 234                 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
 235                 free(service);
 236                 return NULL;
 237         }
 238
 239         for (t = res; t; t = t->ai_next) {
 240                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 241                 if (sockfd >= 0) {
 242                         n = 1;
 243
 244                         setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
 245
 246                         if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
 247                                 break;
 248                         close(sockfd);
 249                         sockfd = -1;
 250                 }
 251         }
 252
 253         freeaddrinfo(res);
 254         free(service);
 255
 256         if (sockfd < 0) {
 257                 fprintf(stderr, "Couldn't listen to port %d\n", port);
 258                 return NULL;
 259         }
 260
 261         listen(sockfd, 1);
 262         connfd = accept(sockfd, NULL, 0);
 263         close(sockfd);
 264         if (connfd < 0) {
 265                 fprintf(stderr, "accept() failed\n");
 266                 return NULL;
 267         }
 268
 269         n = read(connfd, msg, sizeof msg);
 270         if (n != sizeof msg) {
 271                 perror("server read");
 272                 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
 273                 goto out;
 274         }
 275
 276         rem_dest = malloc(sizeof *rem_dest);
 277         if (!rem_dest)
 278                 goto out;
 279
 280         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
 281         wire_gid_to_gid(gid, &rem_dest->gid);
 282
 283         if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) {
 284                 fprintf(stderr, "Couldn't connect to remote QP\n");
 285                 free(rem_dest);
 286                 rem_dest = NULL;
 287                 goto out;
 288         }
 289
 290
 291         gid_to_wire_gid(&my_dest->gid, gid);
 292         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
 293         if (write(connfd, msg, sizeof msg) != sizeof msg) {
 294                 fprintf(stderr, "Couldn't send local address\n");
 295                 free(rem_dest);
 296                 rem_dest = NULL;
 297                 goto out;
 298         }
 299
 300         read(connfd, msg, sizeof msg);
 301
 302 out:
 303         close(connfd);
 304         return rem_dest;
 305 }
 306
 307 #include <sys/param.h>
 308
 309 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 310                                             int rx_depth, int port,
 311                                             int use_event, int is_server)
 312 {
 313         struct pingpong_context *ctx;
 314
 315         ctx = calloc(1, sizeof *ctx);
 316         if (!ctx)
 317                 return NULL;
 318
 319         ctx->size     = size;
 320         ctx->rx_depth = rx_depth;
 321
 322         ctx->buf = malloc(roundup(size, page_size));
 323         if (!ctx->buf) {
 324                 fprintf(stderr, "Couldn't allocate work buf.\n");
 325                 return NULL;
 326         }
 327
 328         memset(ctx->buf, 0x7b + is_server, size);
 329
 330         ctx->context = ibv_open_device(ib_dev);
 331         if (!ctx->context) {
 332                 fprintf(stderr, "Couldn't get context for %s\n",
 333                         ibv_get_device_name(ib_dev));
 334                 return NULL;
 335         }
 336
 337         if (use_event) {
 338                 ctx->channel = ibv_create_comp_channel(ctx->context);
 339                 if (!ctx->channel) {
 340                         fprintf(stderr, "Couldn't create completion channel\n");
 341                         return NULL;
 342                 }
 343         } else
 344                 ctx->channel = NULL;
 345
 346         ctx->pd = ibv_alloc_pd(ctx->context);
 347         if (!ctx->pd) {
 348                 fprintf(stderr, "Couldn't allocate PD\n");
 349                 return NULL;
 350         }
 351
 352         ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE);
 353         if (!ctx->mr) {
 354                 fprintf(stderr, "Couldn't register MR\n");
 355                 return NULL;
 356         }
 357
 358         ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
 359                                 ctx->channel, 0);
 360         if (!ctx->cq) {
 361                 fprintf(stderr, "Couldn't create CQ\n");
 362                 return NULL;
 363         }
 364
 365         {
 366                 struct ibv_qp_init_attr attr = {
 367                         .send_cq = ctx->cq,
 368                         .recv_cq = ctx->cq,
 369                         .cap     = {
 370                                 .max_send_wr  = 1,
 371                                 .max_recv_wr  = rx_depth,
 372                                 .max_send_sge = 1,
 373                                 .max_recv_sge = 1
 374                         },
 375                         .qp_type = IBV_QPT_RC
 376                 };
 377
 378                 ctx->qp = ibv_create_qp(ctx->pd, &attr);
 379                 if (!ctx->qp)  {
 380                         fprintf(stderr, "Couldn't create QP\n");
 381                         return NULL;
 382                 }
 383         }
 384
 385         {
 386                 struct ibv_qp_attr attr = {
 387                         .qp_state        = IBV_QPS_INIT,
 388                         .pkey_index      = 0,
 389                         .port_num        = port,
 390                         .qp_access_flags = 0
 391                 };
 392
 393                 if (ibv_modify_qp(ctx->qp, &attr,
 394                                   IBV_QP_STATE              |
 395                                   IBV_QP_PKEY_INDEX         |
 396                                   IBV_QP_PORT               |
 397                                   IBV_QP_ACCESS_FLAGS)) {
 398                         fprintf(stderr, "Failed to modify QP to INIT\n");
 399                         return NULL;
 400                 }
 401         }
 402
 403         return ctx;
 404 }
 405
 406 int pp_close_ctx(struct pingpong_context *ctx)
 407 {
 408         if (ibv_destroy_qp(ctx->qp)) {
 409                 fprintf(stderr, "Couldn't destroy QP\n");
 410                 return 1;
 411         }
 412
 413         if (ibv_destroy_cq(ctx->cq)) {
 414                 fprintf(stderr, "Couldn't destroy CQ\n");
 415                 return 1;
 416         }
 417
 418         if (ibv_dereg_mr(ctx->mr)) {
 419                 fprintf(stderr, "Couldn't deregister MR\n");
 420                 return 1;
 421         }
 422
 423         if (ibv_dealloc_pd(ctx->pd)) {
 424                 fprintf(stderr, "Couldn't deallocate PD\n");
 425                 return 1;
 426         }
 427
 428         if (ctx->channel) {
 429                 if (ibv_destroy_comp_channel(ctx->channel)) {
 430                         fprintf(stderr, "Couldn't destroy completion channel\n");
 431                         return 1;
 432                 }
 433         }
 434
 435         if (ibv_close_device(ctx->context)) {
 436                 fprintf(stderr, "Couldn't release context\n");
 437                 return 1;
 438         }
 439
 440         free(ctx->buf);
 441         free(ctx);
 442
 443         return 0;
 444 }
 445
 446 static int pp_post_recv(struct pingpong_context *ctx, int n)
 447 {
 448         struct ibv_sge list = {
 449                 .addr   = (uintptr_t) ctx->buf,
 450                 .length = ctx->size,
 451                 .lkey   = ctx->mr->lkey
 452         };
 453         struct ibv_recv_wr wr = {
 454                 .wr_id      = PINGPONG_RECV_WRID,
 455                 .sg_list    = &list,
 456                 .num_sge    = 1,
 457         };
 458         struct ibv_recv_wr *bad_wr;
 459         int i;
 460
 461         for (i = 0; i < n; ++i)
 462                 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
 463                         break;
 464
 465         return i;
 466 }
 467
 468 static int pp_post_send(struct pingpong_context *ctx)
 469 {
 470         struct ibv_sge list = {
 471                 .addr   = (uintptr_t) ctx->buf,
 472                 .length = ctx->size,
 473                 .lkey   = ctx->mr->lkey
 474         };
 475         struct ibv_send_wr wr = {
 476                 .wr_id      = PINGPONG_SEND_WRID,
 477                 .sg_list    = &list,
 478                 .num_sge    = 1,
 479                 .opcode     = IBV_WR_SEND,
 480                 .send_flags = IBV_SEND_SIGNALED,
 481         };
 482         struct ibv_send_wr *bad_wr;
 483
 484         return ibv_post_send(ctx->qp, &wr, &bad_wr);
 485 }
 486
 487 static void usage(const char *argv0)
 488 {
 489         printf("Usage:\n");
 490         printf("  %s            start a server and wait for connection\n", argv0);
 491         printf("  %s <host>     connect to server at <host>\n", argv0);
 492         printf("\n");
 493         printf("Options:\n");
 494         printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
 495         printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
 496         printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
 497         printf("  -s, --size=<size>      size of message to exchange (default 4096)\n");
 498         printf("  -m, --mtu=<size>       path MTU (default 1024)\n");
 499         printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
 500         printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
 501         printf("  -l, --sl=<sl>          service level value\n");
 502         printf("  -e, --events           sleep on CQ events (default poll)\n");
 503         printf("  -g, --gid-idx=<gid index> local port gid index\n");
 504 }
 505
 506 int main(int argc, char *argv[])
 507 {
 508         struct ibv_device      **dev_list;
 509         struct ibv_device       *ib_dev;
 510         struct pingpong_context *ctx;
 511         struct pingpong_dest     my_dest;
 512         struct pingpong_dest    *rem_dest;
 513         struct timeval           start, end;
 514         char                    *ib_devname = NULL;
 515         char                    *servername = NULL;
 516         int                      port = 18515;
 517         int                      ib_port = 1;
 518         int                      size = 4096;
 519         enum ibv_mtu             mtu = IBV_MTU_1024;
 520         int                      rx_depth = 500;
 521         int                      iters = 1000;
 522         int                      use_event = 0;
 523         int                      routs;
 524         int                      rcnt, scnt;
 525         int                      num_cq_events = 0;
 526         int                      sl = 0;
 527         int                      gidx = -1;
 528         char                     gid[33];
 529
 530         srand48(getpid() * time(NULL));
 531
 532         while (1) {
 533                 int c;
 534
 535                 static struct option long_options[] = {
 536                         { .name = "port",     .has_arg = 1, .val = 'p' },
 537                         { .name = "ib-dev",   .has_arg = 1, .val = 'd' },
 538                         { .name = "ib-port",  .has_arg = 1, .val = 'i' },
 539                         { .name = "size",     .has_arg = 1, .val = 's' },
 540                         { .name = "mtu",      .has_arg = 1, .val = 'm' },
 541                         { .name = "rx-depth", .has_arg = 1, .val = 'r' },
 542                         { .name = "iters",    .has_arg = 1, .val = 'n' },
 543                         { .name = "sl",       .has_arg = 1, .val = 'l' },
 544                         { .name = "events",   .has_arg = 0, .val = 'e' },
 545                         { .name = "gid-idx",  .has_arg = 1, .val = 'g' },
 546                         { 0 }
 547                 };
 548
 549                 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL);
 550                 if (c == -1)
 551                         break;
 552
 553                 switch (c) {
 554                 case 'p':
 555                         port = strtol(optarg, NULL, 0);
 556                         if (port < 0 || port > 65535) {
 557                                 usage(argv[0]);
 558                                 return 1;
 559                         }
 560                         break;
 561
 562                 case 'd':
 563                         ib_devname = strdup(optarg);
 564                         break;
 565
 566                 case 'i':
 567                         ib_port = strtol(optarg, NULL, 0);
 568                         if (ib_port < 0) {
 569                                 usage(argv[0]);
 570                                 return 1;
 571                         }
 572                         break;
 573
 574                 case 's':
 575                         size = strtol(optarg, NULL, 0);
 576                         break;
 577
 578                 case 'm':
 579                         mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
 580                         if (mtu < 0) {
 581                                 usage(argv[0]);
 582                                 return 1;
 583                         }
 584                         break;
 585
 586                 case 'r':
 587                         rx_depth = strtol(optarg, NULL, 0);
 588                         break;
 589
 590                 case 'n':
 591                         iters = strtol(optarg, NULL, 0);
 592                         break;
 593
 594                 case 'l':
 595                         sl = strtol(optarg, NULL, 0);
 596                         break;
 597
 598                 case 'e':
 599                         ++use_event;
 600                         break;
 601
 602                 case 'g':
 603                         gidx = strtol(optarg, NULL, 0);
 604                         break;
 605
 606                 default:
 607                         usage(argv[0]);
 608                         return 1;
 609                 }
 610         }
 611
 612         if (optind == argc - 1)
 613                 servername = strdup(argv[optind]);
 614         else if (optind < argc) {
 615                 usage(argv[0]);
 616                 return 1;
 617         }
 618
 619         page_size = sysconf(_SC_PAGESIZE);
 620
 621         dev_list = ibv_get_device_list(NULL);
 622         if (!dev_list) {
 623                 perror("Failed to get IB devices list");
 624                 return 1;
 625         }
 626
 627         if (!ib_devname) {
 628                 ib_dev = *dev_list;
 629                 if (!ib_dev) {
 630                         fprintf(stderr, "No IB devices found\n");
 631                         return 1;
 632                 }
 633         } else {
 634                 int i;
 635                 for (i = 0; dev_list[i]; ++i)
 636                         if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
 637                                 break;
 638                 ib_dev = dev_list[i];
 639                 if (!ib_dev) {
 640                         fprintf(stderr, "IB device %s not found\n", ib_devname);
 641                         return 1;
 642                 }
 643         }
 644
 645         ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event, !servername);
 646         if (!ctx)
 647                 return 1;
 648
 649         routs = pp_post_recv(ctx, ctx->rx_depth);
 650         if (routs < ctx->rx_depth) {
 651                 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
 652                 return 1;
 653         }
 654
 655         if (use_event)
 656                 if (ibv_req_notify_cq(ctx->cq, 0)) {
 657                         fprintf(stderr, "Couldn't request CQ notification\n");
 658                         return 1;
 659                 }
 660
 661
 662         if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
 663                 fprintf(stderr, "Couldn't get port info\n");
 664                 return 1;
 665         }
 666
 667         my_dest.lid = ctx->portinfo.lid;
 668         if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) {
 669                 fprintf(stderr, "Couldn't get local LID\n");
 670                 return 1;
 671         }
 672
 673         if (gidx >= 0) {
 674                 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
 675                         fprintf(stderr, "Could not get local gid for gid index %d\n", gidx);
 676                         return 1;
 677                 }
 678         } else
 679                 memset(&my_dest.gid, 0, sizeof my_dest.gid);
 680
 681         my_dest.qpn = ctx->qp->qp_num;
 682         my_dest.psn = lrand48() & 0xffffff;
 683         inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
 684         printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 685                my_dest.lid, my_dest.qpn, my_dest.psn, gid);
 686
 687
 688         if (servername)
 689                 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 690         else
 691                 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
 692
 693         if (!rem_dest)
 694                 return 1;
 695
 696         inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
 697         printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 698                rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
 699
 700         if (servername)
 701                 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx))
 702                         return 1;
 703
 704         ctx->pending = PINGPONG_RECV_WRID;
 705
 706         if (servername) {
 707                 if (pp_post_send(ctx)) {
 708                         fprintf(stderr, "Couldn't post send\n");
 709                         return 1;
 710                 }
 711                 ctx->pending |= PINGPONG_SEND_WRID;
 712         }
 713
 714         if (gettimeofday(&start, NULL)) {
 715                 perror("gettimeofday");
 716                 return 1;
 717         }
 718
 719         rcnt = scnt = 0;
 720         while (rcnt < iters || scnt < iters) {
 721                 if (use_event) {
 722                         struct ibv_cq *ev_cq;
 723                         void          *ev_ctx;
 724
 725                         if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
 726                                 fprintf(stderr, "Failed to get cq_event\n");
 727                                 return 1;
 728                         }
 729
 730                         ++num_cq_events;
 731
 732                         if (ev_cq != ctx->cq) {
 733                                 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
 734                                 return 1;
 735                         }
 736
 737                         if (ibv_req_notify_cq(ctx->cq, 0)) {
 738                                 fprintf(stderr, "Couldn't request CQ notification\n");
 739                                 return 1;
 740                         }
 741                 }
 742
 743                 {
 744                         struct ibv_wc wc[2];
 745                         int ne, i;
 746
 747                         do {
 748                                 ne = ibv_poll_cq(ctx->cq, 2, wc);
 749                                 if (ne < 0) {
 750                                         fprintf(stderr, "poll CQ failed %d\n", ne);
 751                                         return 1;
 752                                 }
 753
 754                         } while (!use_event && ne < 1);
 755
 756                         for (i = 0; i < ne; ++i) {
 757                                 if (wc[i].status != IBV_WC_SUCCESS) {
 758                                         fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
 759                                                 ibv_wc_status_str(wc[i].status),
 760                                                 wc[i].status, (int) wc[i].wr_id);
 761                                         return 1;
 762                                 }
 763
 764                                 switch ((int) wc[i].wr_id) {
 765                                 case PINGPONG_SEND_WRID:
 766                                         ++scnt;
 767                                         break;
 768
 769                                 case PINGPONG_RECV_WRID:
 770                                         if (--routs <= 1) {
 771                                                 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
 772                                                 if (routs < ctx->rx_depth) {
 773                                                         fprintf(stderr,
 774                                                                 "Couldn't post receive (%d)\n",
 775                                                                 routs);
 776                                                         return 1;
 777                                                 }
 778                                         }
 779
 780                                         ++rcnt;
 781                                         break;
 782
 783                                 default:
 784                                         fprintf(stderr, "Completion for unknown wr_id %d\n",
 785                                                 (int) wc[i].wr_id);
 786                                         return 1;
 787                                 }
 788
 789                                 ctx->pending &= ~(int) wc[i].wr_id;
 790                                 if (scnt < iters && !ctx->pending) {
 791                                         if (pp_post_send(ctx)) {
 792                                                 fprintf(stderr, "Couldn't post send\n");
 793                                                 return 1;
 794                                         }
 795                                         ctx->pending = PINGPONG_RECV_WRID |
 796                                                        PINGPONG_SEND_WRID;
 797                                 }
 798                         }
 799                 }
 800         }
 801
 802         if (gettimeofday(&end, NULL)) {
 803                 perror("gettimeofday");
 804                 return 1;
 805         }
 806
 807         {
 808                 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
 809                         (end.tv_usec - start.tv_usec);
 810                 long long bytes = (long long) size * iters * 2;
 811
 812                 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
 813                        bytes, usec / 1000000., bytes * 8. / usec);
 814                 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
 815                        iters, usec / 1000000., usec / iters);
 816         }
 817
 818         ibv_ack_cq_events(ctx->cq, num_cq_events);
 819
 820         if (pp_close_ctx(ctx))
 821                 return 1;
 822
 823         ibv_free_device_list(dev_list);
 824         free(rem_dest);
 825
 826         return 0;
 827 }