contrib/ofed/libibverbs/examples/ud_pingpong.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #define _GNU_SOURCE
  33 #include <config.h>
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <unistd.h>
  38 #include <string.h>
  39 #include <sys/types.h>
  40 #include <sys/socket.h>
  41 #include <sys/time.h>
  42 #include <netdb.h>
  43 #include <stdlib.h>
  44 #include <getopt.h>
  45 #include <arpa/inet.h>
  46 #include <time.h>
  47
  48 #include "pingpong.h"
  49
  50 enum {
  51         PINGPONG_RECV_WRID = 1,
  52         PINGPONG_SEND_WRID = 2,
  53 };
  54
  55 static int page_size;
  56
  57 struct pingpong_context {
  58         struct ibv_context      *context;
  59         struct ibv_comp_channel *channel;
  60         struct ibv_pd           *pd;
  61         struct ibv_mr           *mr;
  62         struct ibv_cq           *cq;
  63         struct ibv_qp           *qp;
  64         struct ibv_ah           *ah;
  65         void                    *buf;
  66         int                      size;
  67         int                      send_flags;
  68         int                      rx_depth;
  69         int                      pending;
  70         struct ibv_port_attr     portinfo;
  71 };
  72
  73 struct pingpong_dest {
  74         int lid;
  75         int qpn;
  76         int psn;
  77         union ibv_gid gid;
  78 };
  79
  80 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
  81                           int sl, struct pingpong_dest *dest, int sgid_idx)
  82 {
  83         struct ibv_ah_attr ah_attr = {
  84                 .is_global     = 0,
  85                 .dlid          = dest->lid,
  86                 .sl            = sl,
  87                 .src_path_bits = 0,
  88                 .port_num      = port
  89         };
  90         struct ibv_qp_attr attr = {
  91                 .qp_state               = IBV_QPS_RTR
  92         };
  93
  94         if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
  95                 fprintf(stderr, "Failed to modify QP to RTR\n");
  96                 return 1;
  97         }
  98
  99         attr.qp_state       = IBV_QPS_RTS;
 100         attr.sq_psn         = my_psn;
 101
 102         if (ibv_modify_qp(ctx->qp, &attr,
 103                           IBV_QP_STATE              |
 104                           IBV_QP_SQ_PSN)) {
 105                 fprintf(stderr, "Failed to modify QP to RTS\n");
 106                 return 1;
 107         }
 108
 109         if (dest->gid.global.interface_id) {
 110                 ah_attr.is_global = 1;
 111                 ah_attr.grh.hop_limit = 1;
 112                 ah_attr.grh.dgid = dest->gid;
 113                 ah_attr.grh.sgid_index = sgid_idx;
 114         }
 115
 116         ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
 117         if (!ctx->ah) {
 118                 fprintf(stderr, "Failed to create AH\n");
 119                 return 1;
 120         }
 121
 122         return 0;
 123 }
 124
 125 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
 126                                                  const struct pingpong_dest *my_dest)
 127 {
 128         struct addrinfo *res, *t;
 129         struct addrinfo hints = {
 130                 .ai_family   = AF_INET,
 131                 .ai_socktype = SOCK_STREAM
 132         };
 133         char *service;
 134         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 135         int n;
 136         int sockfd = -1;
 137         struct pingpong_dest *rem_dest = NULL;
 138         char gid[33];
 139
 140         if (asprintf(&service, "%d", port) < 0)
 141                 return NULL;
 142
 143         n = getaddrinfo(servername, service, &hints, &res);
 144
 145         if (n < 0) {
 146                 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
 147                 free(service);
 148                 return NULL;
 149         }
 150
 151         for (t = res; t; t = t->ai_next) {
 152                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 153                 if (sockfd >= 0) {
 154                         if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
 155                                 break;
 156                         close(sockfd);
 157                         sockfd = -1;
 158                 }
 159         }
 160
 161         freeaddrinfo_null(res);
 162         free(service);
 163
 164         if (sockfd < 0) {
 165                 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
 166                 return NULL;
 167         }
 168
 169         gid_to_wire_gid(&my_dest->gid, gid);
 170         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
 171                                                         my_dest->psn, gid);
 172         if (write(sockfd, msg, sizeof msg) != sizeof msg) {
 173                 fprintf(stderr, "Couldn't send local address\n");
 174                 goto out;
 175         }
 176
 177         if (read(sockfd, msg, sizeof msg) != sizeof msg ||
 178             write(sockfd, "done", sizeof "done") != sizeof "done") {
 179                 perror("client read/write");
 180                 fprintf(stderr, "Couldn't read/write remote address\n");
 181                 goto out;
 182         }
 183
 184         rem_dest = malloc(sizeof *rem_dest);
 185         if (!rem_dest)
 186                 goto out;
 187
 188         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
 189                                                         &rem_dest->psn, gid);
 190         wire_gid_to_gid(gid, &rem_dest->gid);
 191
 192 out:
 193         close(sockfd);
 194         return rem_dest;
 195 }
 196
 197 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
 198                                                  int ib_port, int port, int sl,
 199                                                  const struct pingpong_dest *my_dest,
 200                                                  int sgid_idx)
 201 {
 202         struct addrinfo *res, *t;
 203         struct addrinfo hints = {
 204                 .ai_flags    = AI_PASSIVE,
 205                 .ai_family   = AF_INET,
 206                 .ai_socktype = SOCK_STREAM
 207         };
 208         char *service;
 209         char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
 210         int n;
 211         int sockfd = -1, connfd;
 212         struct pingpong_dest *rem_dest = NULL;
 213         char gid[33];
 214
 215         if (asprintf(&service, "%d", port) < 0)
 216                 return NULL;
 217
 218         n = getaddrinfo(NULL, service, &hints, &res);
 219
 220         if (n < 0) {
 221                 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
 222                 free(service);
 223                 return NULL;
 224         }
 225
 226         for (t = res; t; t = t->ai_next) {
 227                 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 228                 if (sockfd >= 0) {
 229                         n = 1;
 230
 231                         setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
 232
 233                         if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
 234                                 break;
 235                         close(sockfd);
 236                         sockfd = -1;
 237                 }
 238         }
 239
 240         freeaddrinfo_null(res);
 241         free(service);
 242
 243         if (sockfd < 0) {
 244                 fprintf(stderr, "Couldn't listen to port %d\n", port);
 245                 return NULL;
 246         }
 247
 248         listen(sockfd, 1);
 249         connfd = accept(sockfd, NULL, NULL);
 250         close(sockfd);
 251         if (connfd < 0) {
 252                 fprintf(stderr, "accept() failed\n");
 253                 return NULL;
 254         }
 255
 256         n = read(connfd, msg, sizeof msg);
 257         if (n != sizeof msg) {
 258                 perror("server read");
 259                 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
 260                 goto out;
 261         }
 262
 263         rem_dest = malloc(sizeof *rem_dest);
 264         if (!rem_dest)
 265                 goto out;
 266
 267         sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
 268                                                         &rem_dest->psn, gid);
 269         wire_gid_to_gid(gid, &rem_dest->gid);
 270
 271         if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
 272                                                                 sgid_idx)) {
 273                 fprintf(stderr, "Couldn't connect to remote QP\n");
 274                 free(rem_dest);
 275                 rem_dest = NULL;
 276                 goto out;
 277         }
 278
 279         gid_to_wire_gid(&my_dest->gid, gid);
 280         sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
 281                                                         my_dest->psn, gid);
 282         if (write(connfd, msg, sizeof msg) != sizeof msg ||
 283             read(connfd, msg, sizeof msg) != sizeof "done") {
 284                 fprintf(stderr, "Couldn't send/recv local address\n");
 285                 free(rem_dest);
 286                 rem_dest = NULL;
 287                 goto out;
 288         }
 289 out:
 290         close(connfd);
 291         return rem_dest;
 292 }
 293
 294 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 295                                             int rx_depth, int port,
 296                                             int use_event)
 297 {
 298         struct pingpong_context *ctx;
 299
 300         ctx = malloc(sizeof *ctx);
 301         if (!ctx)
 302                 return NULL;
 303
 304         ctx->size       = size;
 305         ctx->send_flags = IBV_SEND_SIGNALED;
 306         ctx->rx_depth   = rx_depth;
 307
 308         ctx->buf = memalign(page_size, size + 40);
 309         if (!ctx->buf) {
 310                 fprintf(stderr, "Couldn't allocate work buf.\n");
 311                 goto clean_ctx;
 312         }
 313
 314         /* FIXME memset(ctx->buf, 0, size + 40); */
 315         memset(ctx->buf, 0x7b, size + 40);
 316
 317         ctx->context = ibv_open_device(ib_dev);
 318         if (!ctx->context) {
 319                 fprintf(stderr, "Couldn't get context for %s\n",
 320                         ibv_get_device_name(ib_dev));
 321                 goto clean_buffer;
 322         }
 323
 324         {
 325                 struct ibv_port_attr port_info = {};
 326                 int mtu;
 327
 328                 if (ibv_query_port(ctx->context, port, &port_info)) {
 329                         fprintf(stderr, "Unable to query port info for port %d\n", port);
 330                         goto clean_device;
 331                 }
 332                 mtu = 1 << (port_info.active_mtu + 7);
 333                 if (size > mtu) {
 334                         fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
 335                         goto clean_device;
 336                 }
 337         }
 338
 339         if (use_event) {
 340                 ctx->channel = ibv_create_comp_channel(ctx->context);
 341                 if (!ctx->channel) {
 342                         fprintf(stderr, "Couldn't create completion channel\n");
 343                         goto clean_device;
 344                 }
 345         } else
 346                 ctx->channel = NULL;
 347
 348         ctx->pd = ibv_alloc_pd(ctx->context);
 349         if (!ctx->pd) {
 350                 fprintf(stderr, "Couldn't allocate PD\n");
 351                 goto clean_comp_channel;
 352         }
 353
 354         ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
 355         if (!ctx->mr) {
 356                 fprintf(stderr, "Couldn't register MR\n");
 357                 goto clean_pd;
 358         }
 359
 360         ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
 361                                 ctx->channel, 0);
 362         if (!ctx->cq) {
 363                 fprintf(stderr, "Couldn't create CQ\n");
 364                 goto clean_mr;
 365         }
 366
 367         {
 368                 struct ibv_qp_attr attr;
 369                 struct ibv_qp_init_attr init_attr = {
 370                         .send_cq = ctx->cq,
 371                         .recv_cq = ctx->cq,
 372                         .cap     = {
 373                                 .max_send_wr  = 1,
 374                                 .max_recv_wr  = rx_depth,
 375                                 .max_send_sge = 1,
 376                                 .max_recv_sge = 1
 377                         },
 378                         .qp_type = IBV_QPT_UD,
 379                 };
 380
 381                 ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
 382                 if (!ctx->qp)  {
 383                         fprintf(stderr, "Couldn't create QP\n");
 384                         goto clean_cq;
 385                 }
 386
 387                 ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
 388                 if (init_attr.cap.max_inline_data >= size) {
 389                         ctx->send_flags |= IBV_SEND_INLINE;
 390                 }
 391         }
 392
 393         {
 394                 struct ibv_qp_attr attr = {
 395                         .qp_state        = IBV_QPS_INIT,
 396                         .pkey_index      = 0,
 397                         .port_num        = port,
 398                         .qkey            = 0x11111111
 399                 };
 400
 401                 if (ibv_modify_qp(ctx->qp, &attr,
 402                                   IBV_QP_STATE              |
 403                                   IBV_QP_PKEY_INDEX         |
 404                                   IBV_QP_PORT               |
 405                                   IBV_QP_QKEY)) {
 406                         fprintf(stderr, "Failed to modify QP to INIT\n");
 407                         goto clean_qp;
 408                 }
 409         }
 410
 411         return ctx;
 412
 413 clean_qp:
 414         ibv_destroy_qp(ctx->qp);
 415
 416 clean_cq:
 417         ibv_destroy_cq(ctx->cq);
 418
 419 clean_mr:
 420         ibv_dereg_mr(ctx->mr);
 421
 422 clean_pd:
 423         ibv_dealloc_pd(ctx->pd);
 424
 425 clean_comp_channel:
 426         if (ctx->channel)
 427                 ibv_destroy_comp_channel(ctx->channel);
 428
 429 clean_device:
 430         ibv_close_device(ctx->context);
 431
 432 clean_buffer:
 433         free(ctx->buf);
 434
 435 clean_ctx:
 436         free(ctx);
 437
 438         return NULL;
 439 }
 440
 441 static int pp_close_ctx(struct pingpong_context *ctx)
 442 {
 443         if (ibv_destroy_qp(ctx->qp)) {
 444                 fprintf(stderr, "Couldn't destroy QP\n");
 445                 return 1;
 446         }
 447
 448         if (ibv_destroy_cq(ctx->cq)) {
 449                 fprintf(stderr, "Couldn't destroy CQ\n");
 450                 return 1;
 451         }
 452
 453         if (ibv_dereg_mr(ctx->mr)) {
 454                 fprintf(stderr, "Couldn't deregister MR\n");
 455                 return 1;
 456         }
 457
 458         if (ibv_destroy_ah(ctx->ah)) {
 459                 fprintf(stderr, "Couldn't destroy AH\n");
 460                 return 1;
 461         }
 462
 463         if (ibv_dealloc_pd(ctx->pd)) {
 464                 fprintf(stderr, "Couldn't deallocate PD\n");
 465                 return 1;
 466         }
 467
 468         if (ctx->channel) {
 469                 if (ibv_destroy_comp_channel(ctx->channel)) {
 470                         fprintf(stderr, "Couldn't destroy completion channel\n");
 471                         return 1;
 472                 }
 473         }
 474
 475         if (ibv_close_device(ctx->context)) {
 476                 fprintf(stderr, "Couldn't release context\n");
 477                 return 1;
 478         }
 479
 480         free(ctx->buf);
 481         free(ctx);
 482
 483         return 0;
 484 }
 485
 486 static int pp_post_recv(struct pingpong_context *ctx, int n)
 487 {
 488         struct ibv_sge list = {
 489                 .addr   = (uintptr_t) ctx->buf,
 490                 .length = ctx->size + 40,
 491                 .lkey   = ctx->mr->lkey
 492         };
 493         struct ibv_recv_wr wr = {
 494                 .wr_id      = PINGPONG_RECV_WRID,
 495                 .sg_list    = &list,
 496                 .num_sge    = 1,
 497         };
 498         struct ibv_recv_wr *bad_wr;
 499         int i;
 500
 501         for (i = 0; i < n; ++i)
 502                 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
 503                         break;
 504
 505         return i;
 506 }
 507
 508 static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
 509 {
 510         struct ibv_sge list = {
 511                 .addr   = (uintptr_t) ctx->buf + 40,
 512                 .length = ctx->size,
 513                 .lkey   = ctx->mr->lkey
 514         };
 515         struct ibv_send_wr wr = {
 516                 .wr_id      = PINGPONG_SEND_WRID,
 517                 .sg_list    = &list,
 518                 .num_sge    = 1,
 519                 .opcode     = IBV_WR_SEND,
 520                 .send_flags = ctx->send_flags,
 521                 .wr         = {
 522                         .ud = {
 523                                  .ah          = ctx->ah,
 524                                  .remote_qpn  = qpn,
 525                                  .remote_qkey = 0x11111111
 526                          }
 527                 }
 528         };
 529         struct ibv_send_wr *bad_wr;
 530
 531         return ibv_post_send(ctx->qp, &wr, &bad_wr);
 532 }
 533
 534 static void usage(const char *argv0)
 535 {
 536         printf("Usage:\n");
 537         printf("  %s            start a server and wait for connection\n", argv0);
 538         printf("  %s <host>     connect to server at <host>\n", argv0);
 539         printf("\n");
 540         printf("Options:\n");
 541         printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
 542         printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
 543         printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
 544         printf("  -s, --size=<size>      size of message to exchange (default 2048)\n");
 545         printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
 546         printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
 547         printf("  -l, --sl=<SL>          send messages with service level <SL> (default 0)\n");
 548         printf("  -e, --events           sleep on CQ events (default poll)\n");
 549         printf("  -g, --gid-idx=<gid index> local port gid index\n");
 550 }
 551
 552 int main(int argc, char *argv[])
 553 {
 554         struct ibv_device      **dev_list;
 555         struct ibv_device       *ib_dev;
 556         struct pingpong_context *ctx;
 557         struct pingpong_dest     my_dest;
 558         struct pingpong_dest    *rem_dest;
 559         struct timeval           start, end;
 560         char                    *ib_devname = NULL;
 561         char                    *servername = NULL;
 562         unsigned int             port = 18515;
 563         int                      ib_port = 1;
 564         unsigned int             size = 2048;
 565         unsigned int             rx_depth = 500;
 566         unsigned int             iters = 1000;
 567         int                      use_event = 0;
 568         int                      routs;
 569         int                      rcnt, scnt;
 570         int                      num_cq_events = 0;
 571         int                      sl = 0;
 572         int                      gidx = -1;
 573         char                     gid[33];
 574
 575         srand48(getpid() * time(NULL));
 576
 577         while (1) {
 578                 int c;
 579
 580                 static struct option long_options[] = {
 581                         { .name = "port",     .has_arg = 1, .val = 'p' },
 582                         { .name = "ib-dev",   .has_arg = 1, .val = 'd' },
 583                         { .name = "ib-port",  .has_arg = 1, .val = 'i' },
 584                         { .name = "size",     .has_arg = 1, .val = 's' },
 585                         { .name = "rx-depth", .has_arg = 1, .val = 'r' },
 586                         { .name = "iters",    .has_arg = 1, .val = 'n' },
 587                         { .name = "sl",       .has_arg = 1, .val = 'l' },
 588                         { .name = "events",   .has_arg = 0, .val = 'e' },
 589                         { .name = "gid-idx",  .has_arg = 1, .val = 'g' },
 590                         {}
 591                 };
 592
 593                 c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
 594                                                         long_options, NULL);
 595                 if (c == -1)
 596                         break;
 597
 598                 switch (c) {
 599                 case 'p':
 600                         port = strtol(optarg, NULL, 0);
 601                         if (port > 65535) {
 602                                 usage(argv[0]);
 603                                 return 1;
 604                         }
 605                         break;
 606
 607                 case 'd':
 608                         ib_devname = strdupa(optarg);
 609                         break;
 610
 611                 case 'i':
 612                         ib_port = strtol(optarg, NULL, 0);
 613                         if (ib_port < 1) {
 614                                 usage(argv[0]);
 615                                 return 1;
 616                         }
 617                         break;
 618
 619                 case 's':
 620                         size = strtoul(optarg, NULL, 0);
 621                         break;
 622
 623                 case 'r':
 624                         rx_depth = strtoul(optarg, NULL, 0);
 625                         break;
 626
 627                 case 'n':
 628                         iters = strtoul(optarg, NULL, 0);
 629                         break;
 630
 631                 case 'l':
 632                         sl = strtol(optarg, NULL, 0);
 633                         break;
 634
 635                 case 'e':
 636                         ++use_event;
 637                         break;
 638
 639                 case 'g':
 640                         gidx = strtol(optarg, NULL, 0);
 641                         break;
 642
 643                 default:
 644                         usage(argv[0]);
 645                         return 1;
 646                 }
 647         }
 648
 649         if (optind == argc - 1)
 650                 servername = strdupa(argv[optind]);
 651         else if (optind < argc) {
 652                 usage(argv[0]);
 653                 return 1;
 654         }
 655
 656         page_size = sysconf(_SC_PAGESIZE);
 657
 658         dev_list = ibv_get_device_list(NULL);
 659         if (!dev_list) {
 660                 perror("Failed to get IB devices list");
 661                 return 1;
 662         }
 663
 664         if (!ib_devname) {
 665                 ib_dev = *dev_list;
 666                 if (!ib_dev) {
 667                         fprintf(stderr, "No IB devices found\n");
 668                         return 1;
 669                 }
 670         } else {
 671                 int i;
 672                 for (i = 0; dev_list[i]; ++i)
 673                         if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
 674                                 break;
 675                 ib_dev = dev_list[i];
 676                 if (!ib_dev) {
 677                         fprintf(stderr, "IB device %s not found\n", ib_devname);
 678                         return 1;
 679                 }
 680         }
 681
 682         ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
 683         if (!ctx)
 684                 return 1;
 685
 686         routs = pp_post_recv(ctx, ctx->rx_depth);
 687         if (routs < ctx->rx_depth) {
 688                 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
 689                 return 1;
 690         }
 691
 692         if (use_event)
 693                 if (ibv_req_notify_cq(ctx->cq, 0)) {
 694                         fprintf(stderr, "Couldn't request CQ notification\n");
 695                         return 1;
 696                 }
 697
 698         if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
 699                 fprintf(stderr, "Couldn't get port info\n");
 700                 return 1;
 701         }
 702         my_dest.lid = ctx->portinfo.lid;
 703
 704         my_dest.qpn = ctx->qp->qp_num;
 705         my_dest.psn = lrand48() & 0xffffff;
 706
 707         if (gidx >= 0) {
 708                 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
 709                         fprintf(stderr, "Could not get local gid for gid index "
 710                                                                 "%d\n", gidx);
 711                         return 1;
 712                 }
 713         } else
 714                 memset(&my_dest.gid, 0, sizeof my_dest.gid);
 715
 716         inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
 717         printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
 718                my_dest.lid, my_dest.qpn, my_dest.psn, gid);
 719
 720         if (servername)
 721                 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 722         else
 723                 rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl,
 724                                                         &my_dest, gidx);
 725
 726         if (!rem_dest)
 727                 return 1;
 728
 729         inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
 730         printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
 731                rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
 732
 733         if (servername)
 734                 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest,
 735                                                                         gidx))
 736                         return 1;
 737
 738         ctx->pending = PINGPONG_RECV_WRID;
 739
 740         if (servername) {
 741                 if (pp_post_send(ctx, rem_dest->qpn)) {
 742                         fprintf(stderr, "Couldn't post send\n");
 743                         return 1;
 744                 }
 745                 ctx->pending |= PINGPONG_SEND_WRID;
 746         }
 747
 748         if (gettimeofday(&start, NULL)) {
 749                 perror("gettimeofday");
 750                 return 1;
 751         }
 752
 753         rcnt = scnt = 0;
 754         while (rcnt < iters || scnt < iters) {
 755                 if (use_event) {
 756                         struct ibv_cq *ev_cq;
 757                         void          *ev_ctx;
 758
 759                         if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
 760                                 fprintf(stderr, "Failed to get cq_event\n");
 761                                 return 1;
 762                         }
 763
 764                         ++num_cq_events;
 765
 766                         if (ev_cq != ctx->cq) {
 767                                 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
 768                                 return 1;
 769                         }
 770
 771                         if (ibv_req_notify_cq(ctx->cq, 0)) {
 772                                 fprintf(stderr, "Couldn't request CQ notification\n");
 773                                 return 1;
 774                         }
 775                 }
 776
 777                 {
 778                         struct ibv_wc wc[2];
 779                         int ne, i;
 780
 781                         do {
 782                                 ne = ibv_poll_cq(ctx->cq, 2, wc);
 783                                 if (ne < 0) {
 784                                         fprintf(stderr, "poll CQ failed %d\n", ne);
 785                                         return 1;
 786                                 }
 787                         } while (!use_event && ne < 1);
 788
 789                         for (i = 0; i < ne; ++i) {
 790                                 if (wc[i].status != IBV_WC_SUCCESS) {
 791                                         fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
 792                                                 ibv_wc_status_str(wc[i].status),
 793                                                 wc[i].status, (int) wc[i].wr_id);
 794                                         return 1;
 795                                 }
 796
 797                                 switch ((int) wc[i].wr_id) {
 798                                 case PINGPONG_SEND_WRID:
 799                                         ++scnt;
 800                                         break;
 801
 802                                 case PINGPONG_RECV_WRID:
 803                                         if (--routs <= 1) {
 804                                                 routs += pp_post_recv(ctx, ctx->rx_depth - routs);
 805                                                 if (routs < ctx->rx_depth) {
 806                                                         fprintf(stderr,
 807                                                                 "Couldn't post receive (%d)\n",
 808                                                                 routs);
 809                                                         return 1;
 810                                                 }
 811                                         }
 812
 813                                         ++rcnt;
 814                                         break;
 815
 816                                 default:
 817                                         fprintf(stderr, "Completion for unknown wr_id %d\n",
 818                                                 (int) wc[i].wr_id);
 819                                         return 1;
 820                                 }
 821
 822                                 ctx->pending &= ~(int) wc[i].wr_id;
 823                                 if (scnt < iters && !ctx->pending) {
 824                                         if (pp_post_send(ctx, rem_dest->qpn)) {
 825                                                 fprintf(stderr, "Couldn't post send\n");
 826                                                 return 1;
 827                                         }
 828                                         ctx->pending = PINGPONG_RECV_WRID |
 829                                                        PINGPONG_SEND_WRID;
 830                                 }
 831                         }
 832                 }
 833         }
 834
 835         if (gettimeofday(&end, NULL)) {
 836                 perror("gettimeofday");
 837                 return 1;
 838         }
 839
 840         {
 841                 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
 842                         (end.tv_usec - start.tv_usec);
 843                 long long bytes = (long long) size * iters * 2;
 844
 845                 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
 846                        bytes, usec / 1000000., bytes * 8. / usec);
 847                 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
 848                        iters, usec / 1000000., usec / iters);
 849         }
 850
 851         ibv_ack_cq_events(ctx->cq, num_cq_events);
 852
 853         if (pp_close_ctx(ctx))
 854                 return 1;
 855
 856         ibv_free_device_list(dev_list);
 857         free(rem_dest);
 858
 859         return 0;
 860 }