contrib/ofed/libibverbs/examples/uc_pingpong.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #define _GNU_SOURCE
  33 #include <config.h>
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <unistd.h>
  38 #include <string.h>
  39 #include <sys/types.h>
  40 #include <sys/socket.h>
  41 #include <sys/time.h>
  42 #include <netdb.h>
  43 #include <stdlib.h>
  44 #include <getopt.h>
  45 #include <arpa/inet.h>
  46 #include <time.h>
  47
  48 #include "pingpong.h"
  49
  50 enum {
  51         PINGPONG_RECV_WRID = 1,
  52         PINGPONG_SEND_WRID = 2,
  53 };
  54
  55 static int page_size;
  56
  57 struct pingpong_context {
  58         struct ibv_context      *context;
  59         struct ibv_comp_channel *channel;
  60         struct ibv_pd           *pd;
  61         struct ibv_mr           *mr;
  62         struct ibv_cq           *cq;
  63         struct ibv_qp           *qp;
  64         void                    *buf;
  65         int                      size;
  66         int                      send_flags;
  67         int                      rx_depth;
  68         int                      pending;
  69         struct ibv_port_attr     portinfo;
  70 };
  71
  72 struct pingpong_dest {
  73         int lid;
  74         int qpn;
  75         int psn;
  76         union ibv_gid gid;
  77 };
  78
  79 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
  80                           enum ibv_mtu mtu, int sl,
  81                           struct pingpong_dest *dest, int sgid_idx)
  82 {
  83         struct ibv_qp_attr attr = {
  84                 .qp_state               = IBV_QPS_RTR,
  85                 .path_mtu               = mtu,
  86                 .dest_qp_num            = dest->qpn,
  87                 .rq_psn         = dest->psn,
  88                 .ah_attr                = {
  89                         .is_global      = 0,
  90                         .dlid           = dest->lid,
  91                         .sl             = sl,
  92                         .src_path_bits  = 0,
  93                         .port_num       = port
  94                 }
  95         };
  96
  97         if (dest->gid.global.interface_id) {
  98                 attr.ah_attr.is_global = 1;
  99                 attr.ah_attr.grh.hop_limit = 1;
 100                 attr.ah_attr.grh.dgid = dest->gid;
 101                 attr.ah_attr.grh.sgid_index = sgid_idx;
 102         }
 103         if (ibv_modify_qp(ctx->qp, &attr,
 104                           IBV_QP_STATE              |
 105                           IBV_QP_AV                 |
 106                           IBV_QP_PATH_MTU           |
 107                           IBV_QP_DEST_QPN           |
 108                           IBV_QP_RQ_PSN)) {
 109                 fprintf(stderr, "Failed to modify QP to RTR\n");
 110                 return 1;
 111         }
 112
 113         attr.qp_state       = IBV_QPS_RTS;
 114         attr.sq_psn         = my_psn;
 115         if (ibv_modify_qp(ctx->qp, &attr,
 116                           IBV_QP_STATE              |
 117                           IBV_QP_SQ_PSN)) {
 118                 fprintf(stderr, "Failed to modify QP to RTS\n");
 119                 return 1;
 120         }
 121
 122         return 0;
 123 }
 124
 125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 126                                                  const struct pingpong_dest *my_dest)
 127 {
 128         struct addrinfo *res, *t;
 129         struct addrinfo hints = {
 130                 .ai_family   = AF_UNSPEC,
 131                 .ai_socktype = SOCK_STREAM
 132         };
 133         char *service;
 134         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 135         int n;
 136         int sockfd = -1;
 137         struct pingpong_dest *rem_dest = NULL;
 138         char gid[33];
 139
 140         if (asprintf(&service, "%d", port) < 0)
 141                 return NULL;
 142
 143         n = getaddrinfo(servername, service, &hints, &res);
 144
 145         if (n < 0) {
 146                 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
 147                 free(service);
 148                 return NULL;
 149         }
 150
 151         for (t = res; t; t = t->ai_next) {
 152                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 153                 if (sockfd >= 0) {
 154                         if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
 155                                 break;
 156                         close(sockfd);
 157                         sockfd = -1;
 158                 }
 159         }
 160
 161         freeaddrinfo_null(res);
 162         free(service);
 163
 164         if (sockfd < 0) {
 165                 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
 166                 return NULL;
 167         }
 168
 169         gid_to_wire_gid(&my_dest->gid, gid);
 170         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
 171                                                         my_dest->psn, gid);
 172         if (write(sockfd, msg, sizeof msg) != sizeof msg) {
 173                 fprintf(stderr, "Couldn't send local address\n");
 174                 goto out;
 175         }
 176
 177         if (read(sockfd, msg, sizeof msg) != sizeof msg ||
 178             write(sockfd, "done", sizeof "done") != sizeof "done") {
 179                 perror("client read/write");
 180                 fprintf(stderr, "Couldn't read/write remote address\n");
 181                 goto out;
 182         }
 183
 184
 185         rem_dest = malloc(sizeof *rem_dest);
 186         if (!rem_dest)
 187                 goto out;
 188
 189         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
 190                                                 &rem_dest->psn, gid);
 191         wire_gid_to_gid(gid, &rem_dest->gid);
 192
 193 out:
 194         close(sockfd);
 195         return rem_dest;
 196 }
 197
 198 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
 199                                                  int ib_port, enum ibv_mtu mtu,
 200                                                  int port, int sl,
 201                                                  const struct pingpong_dest *my_dest,
 202                                                  int sgid_idx)
 203 {
 204         struct addrinfo *res, *t;
 205         struct addrinfo hints = {
 206                 .ai_flags    = AI_PASSIVE,
 207                 .ai_family   = AF_INET,
 208                 .ai_socktype = SOCK_STREAM
 209         };
 210         char *service;
 211         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 212         int n;
 213         int sockfd = -1, connfd;
 214         struct pingpong_dest *rem_dest = NULL;
 215         char gid[33];
 216
 217         if (asprintf(&service, "%d", port) < 0)
 218                 return NULL;
 219
 220         n = getaddrinfo(NULL, service, &hints, &res);
 221
 222         if (n < 0) {
 223                 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
 224                 free(service);
 225                 return NULL;
 226         }
 227
 228         for (t = res; t; t = t->ai_next) {
 229                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 230                 if (sockfd >= 0) {
 231                         n = 1;
 232
 233                         setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
 234
 235                         if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
 236                                 break;
 237                         close(sockfd);
 238                         sockfd = -1;
 239                 }
 240         }
 241
 242         freeaddrinfo_null(res);
 243         free(service);
 244
 245         if (sockfd < 0) {
 246                 fprintf(stderr, "Couldn't listen to port %d\n", port);
 247                 return NULL;
 248         }
 249
 250         if (listen(sockfd, 1) < 0) {
 251                 perror("listen() failed");
 252                 close(sockfd);
 253                 return NULL;
 254         }
 255         connfd = accept(sockfd, NULL, NULL);
 256         close(sockfd);
 257         if (connfd < 0) {
 258                 fprintf(stderr, "accept() failed\n");
 259                 return NULL;
 260         }
 261
 262         n = read(connfd, msg, sizeof msg);
 263         if (n != sizeof msg) {
 264                 perror("server read");
 265                 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
 266                 goto out;
 267         }
 268
 269         rem_dest = malloc(sizeof *rem_dest);
 270         if (!rem_dest)
 271                 goto out;
 272
 273         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
 274                                                         &rem_dest->psn, gid);
 275         wire_gid_to_gid(gid, &rem_dest->gid);
 276
 277         if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest,
 278                                                                 sgid_idx)) {
 279                 fprintf(stderr, "Couldn't connect to remote QP\n");
 280                 free(rem_dest);
 281                 rem_dest = NULL;
 282                 goto out;
 283         }
 284
 285
 286         gid_to_wire_gid(&my_dest->gid, gid);
 287         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
 288                                                         my_dest->psn, gid);
 289         if (write(connfd, msg, sizeof msg) != sizeof msg ||
 290             read(connfd, msg, sizeof msg) != sizeof "done") {
 291                 fprintf(stderr, "Couldn't send/recv local address\n");
 292                 free(rem_dest);
 293                 rem_dest = NULL;
 294                 goto out;
 295         }
 296
 297 out:
 298         close(connfd);
 299         return rem_dest;
 300 }
 301
 302 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 303                                             int rx_depth, int port,
 304                                             int use_event)
 305 {
 306         struct pingpong_context *ctx;
 307
 308         ctx = calloc(1, sizeof *ctx);
 309         if (!ctx)
 310                 return NULL;
 311
 312         ctx->size       = size;
 313         ctx->send_flags = IBV_SEND_SIGNALED;
 314         ctx->rx_depth   = rx_depth;
 315
 316         ctx->buf = memalign(page_size, size);
 317         if (!ctx->buf) {
 318                 fprintf(stderr, "Couldn't allocate work buf.\n");
 319                 goto clean_ctx;
 320         }
 321
 322         /* FIXME memset(ctx->buf, 0, size); */
 323         memset(ctx->buf, 0x7b, size);
 324
 325         ctx->context = ibv_open_device(ib_dev);
 326         if (!ctx->context) {
 327                 fprintf(stderr, "Couldn't get context for %s\n",
 328                         ibv_get_device_name(ib_dev));
 329                 goto clean_buffer;
 330         }
 331
 332         if (use_event) {
 333                 ctx->channel = ibv_create_comp_channel(ctx->context);
 334                 if (!ctx->channel) {
 335                         fprintf(stderr, "Couldn't create completion channel\n");
 336                         goto clean_device;
 337                 }
 338         } else
 339                 ctx->channel = NULL;
 340
 341         ctx->pd = ibv_alloc_pd(ctx->context);
 342         if (!ctx->pd) {
 343                 fprintf(stderr, "Couldn't allocate PD\n");
 344                 goto clean_comp_channel;
 345         }
 346
 347         ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE);
 348         if (!ctx->mr) {
 349                 fprintf(stderr, "Couldn't register MR\n");
 350                 goto clean_pd;
 351         }
 352
 353         ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
 354                                 ctx->channel, 0);
 355         if (!ctx->cq) {
 356                 fprintf(stderr, "Couldn't create CQ\n");
 357                 goto clean_mr;
 358         }
 359
 360         {
 361                 struct ibv_qp_attr attr;
 362                 struct ibv_qp_init_attr init_attr = {
 363                         .send_cq = ctx->cq,
 364                         .recv_cq = ctx->cq,
 365                         .cap     = {
 366                                 .max_send_wr  = 1,
 367                                 .max_recv_wr  = rx_depth,
 368                                 .max_send_sge = 1,
 369                                 .max_recv_sge = 1
 370                         },
 371                         .qp_type = IBV_QPT_UC
 372                 };
 373
 374                 ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
 375                 if (!ctx->qp)  {
 376                         fprintf(stderr, "Couldn't create QP\n");
 377                         goto clean_cq;
 378                 }
 379                 ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
 380                 if (init_attr.cap.max_inline_data >= size) {
 381                         ctx->send_flags |= IBV_SEND_INLINE;
 382                 }
 383         }
 384
 385         {
 386                 struct ibv_qp_attr attr = {
 387                         .qp_state        = IBV_QPS_INIT,
 388                         .pkey_index      = 0,
 389                         .port_num        = port,
 390                         .qp_access_flags = 0
 391                 };
 392
 393                 if (ibv_modify_qp(ctx->qp, &attr,
 394                                   IBV_QP_STATE              |
 395                                   IBV_QP_PKEY_INDEX         |
 396                                   IBV_QP_PORT               |
 397                                   IBV_QP_ACCESS_FLAGS)) {
 398                         fprintf(stderr, "Failed to modify QP to INIT\n");
 399                         goto clean_qp;
 400                 }
 401         }
 402
 403         return ctx;
 404
 405 clean_qp:
 406         ibv_destroy_qp(ctx->qp);
 407
 408 clean_cq:
 409         ibv_destroy_cq(ctx->cq);
 410
 411 clean_mr:
 412         ibv_dereg_mr(ctx->mr);
 413
 414 clean_pd:
 415         ibv_dealloc_pd(ctx->pd);
 416
 417 clean_comp_channel:
 418         if (ctx->channel)
 419                 ibv_destroy_comp_channel(ctx->channel);
 420
 421 clean_device:
 422         ibv_close_device(ctx->context);
 423
 424 clean_buffer:
 425         free(ctx->buf);
 426
 427 clean_ctx:
 428         free(ctx);
 429
 430         return NULL;
 431 }
 432
 433 static int pp_close_ctx(struct pingpong_context *ctx)
 434 {
 435         if (ibv_destroy_qp(ctx->qp)) {
 436                 fprintf(stderr, "Couldn't destroy QP\n");
 437                 return 1;
 438         }
 439
 440         if (ibv_destroy_cq(ctx->cq)) {
 441                 fprintf(stderr, "Couldn't destroy CQ\n");
 442                 return 1;
 443         }
 444
 445         if (ibv_dereg_mr(ctx->mr)) {
 446                 fprintf(stderr, "Couldn't deregister MR\n");
 447                 return 1;
 448         }
 449
 450         if (ibv_dealloc_pd(ctx->pd)) {
 451                 fprintf(stderr, "Couldn't deallocate PD\n");
 452                 return 1;
 453         }
 454
 455         if (ctx->channel) {
 456                 if (ibv_destroy_comp_channel(ctx->channel)) {
 457                         fprintf(stderr, "Couldn't destroy completion channel\n");
 458                         return 1;
 459                 }
 460         }
 461
 462         if (ibv_close_device(ctx->context)) {
 463                 fprintf(stderr, "Couldn't release context\n");
 464                 return 1;
 465         }
 466
 467         free(ctx->buf);
 468         free(ctx);
 469
 470         return 0;
 471 }
 472
 473 static int pp_post_recv(struct pingpong_context *ctx, int n)
 474 {
 475         struct ibv_sge list = {
 476                 .addr   = (uintptr_t) ctx->buf,
 477                 .length = ctx->size,
 478                 .lkey   = ctx->mr->lkey
 479         };
 480         struct ibv_recv_wr wr = {
 481                 .wr_id      = PINGPONG_RECV_WRID,
 482                 .sg_list    = &list,
 483                 .num_sge    = 1,
 484         };
 485         struct ibv_recv_wr *bad_wr;
 486         int i;
 487
 488         for (i = 0; i < n; ++i)
 489                 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
 490                         break;
 491
 492         return i;
 493 }
 494
 495 static int pp_post_send(struct pingpong_context *ctx)
 496 {
 497         struct ibv_sge list = {
 498                 .addr   = (uintptr_t) ctx->buf,
 499                 .length = ctx->size,
 500                 .lkey   = ctx->mr->lkey
 501         };
 502         struct ibv_send_wr wr = {
 503                 .wr_id      = PINGPONG_SEND_WRID,
 504                 .sg_list    = &list,
 505                 .num_sge    = 1,
 506                 .opcode     = IBV_WR_SEND,
 507                 .send_flags = ctx->send_flags,
 508         };
 509         struct ibv_send_wr *bad_wr;
 510
 511         return ibv_post_send(ctx->qp, &wr, &bad_wr);
 512 }
 513
 514 static void usage(const char *argv0)
 515 {
 516         printf("Usage:\n");
 517         printf("  %s            start a server and wait for connection\n", argv0);
 518         printf("  %s <host>     connect to server at <host>\n", argv0);
 519         printf("\n");
 520         printf("Options:\n");
 521         printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
 522         printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
 523         printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
 524         printf("  -s, --size=<size>      size of message to exchange (default 4096)\n");
 525         printf("  -m, --mtu=<size>       path MTU (default 1024)\n");
 526         printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
 527         printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
 528         printf("  -l, --sl=<sl>          service level value\n");
 529         printf("  -e, --events           sleep on CQ events (default poll)\n");
 530         printf("  -g, --gid-idx=<gid index> local port gid index\n");
 531 }
 532
 533 int main(int argc, char *argv[])
 534 {
 535         struct ibv_device      **dev_list;
 536         struct ibv_device       *ib_dev;
 537         struct pingpong_context *ctx;
 538         struct pingpong_dest     my_dest;
 539         struct pingpong_dest    *rem_dest;
 540         struct timeval           start, end;
 541         char                    *ib_devname = NULL;
 542         char                    *servername = NULL;
 543         unsigned int             port = 18515;
 544         int                      ib_port = 1;
 545         unsigned int             size = 4096;
 546         enum ibv_mtu             mtu = IBV_MTU_1024;
 547         unsigned int             rx_depth = 500;
 548         unsigned int             iters = 1000;
 549         int                      use_event = 0;
 550         int                      routs;
 551         int                      rcnt, scnt;
 552         int                      num_cq_events = 0;
 553         int                      sl = 0;
 554         int                      gidx = -1;
 555         char                     gid[33];
 556
 557         srand48(getpid() * time(NULL));
 558
 559         while (1) {
 560                 int c;
 561
 562                 static struct option long_options[] = {
 563                         { .name = "port",     .has_arg = 1, .val = 'p' },
 564                         { .name = "ib-dev",   .has_arg = 1, .val = 'd' },
 565                         { .name = "ib-port",  .has_arg = 1, .val = 'i' },
 566                         { .name = "size",     .has_arg = 1, .val = 's' },
 567                         { .name = "mtu",      .has_arg = 1, .val = 'm' },
 568                         { .name = "rx-depth", .has_arg = 1, .val = 'r' },
 569                         { .name = "iters",    .has_arg = 1, .val = 'n' },
 570                         { .name = "sl",       .has_arg = 1, .val = 'l' },
 571                         { .name = "events",   .has_arg = 0, .val = 'e' },
 572                         { .name = "gid-idx",  .has_arg = 1, .val = 'g' },
 573                         {}
 574                 };
 575
 576                 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:",
 577                                                         long_options, NULL);
 578                 if (c == -1)
 579                         break;
 580
 581                 switch (c) {
 582                 case 'p':
 583                         port = strtoul(optarg, NULL, 0);
 584                         if (port > 65535) {
 585                                 usage(argv[0]);
 586                                 return 1;
 587                         }
 588                         break;
 589
 590                 case 'd':
 591                         ib_devname = strdupa(optarg);
 592                         break;
 593
 594                 case 'i':
 595                         ib_port = strtol(optarg, NULL, 0);
 596                         if (ib_port < 1) {
 597                                 usage(argv[0]);
 598                                 return 1;
 599                         }
 600                         break;
 601
 602                 case 's':
 603                         size = strtoul(optarg, NULL, 0);
 604                         break;
 605
 606                 case 'm':
 607                         mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
 608                         if (mtu == 0) {
 609                                 usage(argv[0]);
 610                                 return 1;
 611                         }
 612                         break;
 613
 614                 case 'r':
 615                         rx_depth = strtoul(optarg, NULL, 0);
 616                         break;
 617
 618                 case 'n':
 619                         iters = strtoul(optarg, NULL, 0);
 620                         break;
 621
 622                 case 'l':
 623                         sl = strtol(optarg, NULL, 0);
 624                         break;
 625
 626                 case 'e':
 627                         ++use_event;
 628                         break;
 629
 630                 case 'g':
 631                         gidx = strtol(optarg, NULL, 0);
 632                         break;
 633
 634                 default:
 635                         usage(argv[0]);
 636                         return 1;
 637                 }
 638         }
 639
 640         if (optind == argc - 1)
 641                 servername = strdupa(argv[optind]);
 642         else if (optind < argc) {
 643                 usage(argv[0]);
 644                 return 1;
 645         }
 646
 647         page_size = sysconf(_SC_PAGESIZE);
 648
 649         dev_list = ibv_get_device_list(NULL);
 650         if (!dev_list) {
 651                 perror("Failed to get IB devices list");
 652                 return 1;
 653         }
 654
 655         if (!ib_devname) {
 656                 ib_dev = *dev_list;
 657                 if (!ib_dev) {
 658                         fprintf(stderr, "No IB devices found\n");
 659                         return 1;
 660                 }
 661         } else {
 662                 int i;
 663                 for (i = 0; dev_list[i]; ++i)
 664                         if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
 665                                 break;
 666                 ib_dev = dev_list[i];
 667                 if (!ib_dev) {
 668                         fprintf(stderr, "IB device %s not found\n", ib_devname);
 669                         return 1;
 670                 }
 671         }
 672
 673         ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
 674         if (!ctx)
 675                 return 1;
 676
 677         routs = pp_post_recv(ctx, ctx->rx_depth);
 678         if (routs < ctx->rx_depth) {
 679                 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
 680                 return 1;
 681         }
 682
 683         if (use_event)
 684                 if (ibv_req_notify_cq(ctx->cq, 0)) {
 685                         fprintf(stderr, "Couldn't request CQ notification\n");
 686                         return 1;
 687                 }
 688
 689
 690         if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
 691                 fprintf(stderr, "Couldn't get port info\n");
 692                 return 1;
 693         }
 694
 695         my_dest.lid = ctx->portinfo.lid;
 696         if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET &&
 697                                                         !my_dest.lid) {
 698                 fprintf(stderr, "Couldn't get local LID\n");
 699                 return 1;
 700         }
 701
 702         if (gidx >= 0) {
 703                 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
 704                         fprintf(stderr, "can't read sgid of index %d\n", gidx);
 705                         return 1;
 706                 }
 707         } else
 708                 memset(&my_dest.gid, 0, sizeof my_dest.gid);
 709
 710         my_dest.qpn = ctx->qp->qp_num;
 711         my_dest.psn = lrand48() & 0xffffff;
 712         inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
 713         printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 714                my_dest.lid, my_dest.qpn, my_dest.psn, gid);
 715
 716
 717         if (servername)
 718                 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 719         else
 720                 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl,
 721                                                                 &my_dest, gidx);
 722
 723         if (!rem_dest)
 724                 return 1;
 725
 726         inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
 727         printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 728                rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
 729
 730         if (servername)
 731                 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest,
 732                                         gidx))
 733                         return 1;
 734
 735         ctx->pending = PINGPONG_RECV_WRID;
 736
 737         if (servername) {
 738                 if (pp_post_send(ctx)) {
 739                         fprintf(stderr, "Couldn't post send\n");
 740                         return 1;
 741                 }
 742                 ctx->pending |= PINGPONG_SEND_WRID;
 743         }
 744
 745         if (gettimeofday(&start, NULL)) {
 746                 perror("gettimeofday");
 747                 return 1;
 748         }
 749
 750         rcnt = scnt = 0;
 751         while (rcnt < iters || scnt < iters) {
 752                 if (use_event) {
 753                         struct ibv_cq *ev_cq;
 754                         void          *ev_ctx;
 755
 756                         if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
 757                                 fprintf(stderr, "Failed to get cq_event\n");
 758                                 return 1;
 759                         }
 760
 761                         ++num_cq_events;
 762
 763                         if (ev_cq != ctx->cq) {
 764                                 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
 765                                 return 1;
 766                         }
 767
 768                         if (ibv_req_notify_cq(ctx->cq, 0)) {
 769                                 fprintf(stderr, "Couldn't request CQ notification\n");
 770                                 return 1;
 771                         }
 772                 }
 773
 774                 {
 775                         struct ibv_wc wc[2];
 776                         int ne, i;
 777
 778                         do {
 779                                 ne = ibv_poll_cq(ctx->cq, 2, wc);
 780                                 if (ne < 0) {
 781                                         fprintf(stderr, "poll CQ failed %d\n", ne);
 782                                         return 1;
 783                                 }
 784
 785                         } while (!use_event && ne < 1);
 786
 787                         for (i = 0; i < ne; ++i) {
 788                                 if (wc[i].status != IBV_WC_SUCCESS) {
 789                                         fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
 790                                                 ibv_wc_status_str(wc[i].status),
 791                                                 wc[i].status, (int) wc[i].wr_id);
 792                                         return 1;
 793                                 }
 794
 795                                 switch ((int) wc[i].wr_id) {
 796                                 case PINGPONG_SEND_WRID:
 797                                         ++scnt;
 798                                         break;
 799
 800                                 case PINGPONG_RECV_WRID:
 801                                         if (--routs <= 1) {
 802                                                 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
 803                                                 if (routs < ctx->rx_depth) {
 804                                                         fprintf(stderr,
 805                                                                 "Couldn't post receive (%d)\n",
 806                                                                 routs);
 807                                                         return 1;
 808                                                 }
 809                                         }
 810
 811                                         ++rcnt;
 812                                         break;
 813
 814                                 default:
 815                                         fprintf(stderr, "Completion for unknown wr_id %d\n",
 816                                                 (int) wc[i].wr_id);
 817                                         return 1;
 818                                 }
 819
 820                                 ctx->pending &= ~(int) wc[i].wr_id;
 821                                 if (scnt < iters && !ctx->pending) {
 822                                         if (pp_post_send(ctx)) {
 823                                                 fprintf(stderr, "Couldn't post send\n");
 824                                                 return 1;
 825                                         }
 826                                         ctx->pending = PINGPONG_RECV_WRID |
 827                                                        PINGPONG_SEND_WRID;
 828                                 }
 829                         }
 830                 }
 831         }
 832
 833         if (gettimeofday(&end, NULL)) {
 834                 perror("gettimeofday");
 835                 return 1;
 836         }
 837
 838         {
 839                 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
 840                         (end.tv_usec - start.tv_usec);
 841                 long long bytes = (long long) size * iters * 2;
 842
 843                 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
 844                        bytes, usec / 1000000., bytes * 8. / usec);
 845                 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
 846                        iters, usec / 1000000., usec / iters);
 847         }
 848
 849         ibv_ack_cq_events(ctx->cq, num_cq_events);
 850
 851         if (pp_close_ctx(ctx))
 852                 return 1;
 853
 854         ibv_free_device_list(dev_list);
 855         free(rem_dest);
 856
 857         return 0;
 858 }