contrib/ofed/librdmacm/examples/udaddy.c

   1 /*
   2  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  * $Id$
  33  */
  34
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <stdio.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <netinet/in.h>
  41 #include <sys/socket.h>
  42 #include <netdb.h>
  43 #include <byteswap.h>
  44 #include <getopt.h>
  45
  46 #include <rdma/rdma_cma.h>
  47
  48 struct cmatest_node {
  49         int                     id;
  50         struct rdma_cm_id       *cma_id;
  51         int                     connected;
  52         struct ibv_pd           *pd;
  53         struct ibv_cq           *cq;
  54         struct ibv_mr           *mr;
  55         struct ibv_ah           *ah;
  56         uint32_t                remote_qpn;
  57         uint32_t                remote_qkey;
  58         void                    *mem;
  59 };
  60
  61 struct cmatest {
  62         struct rdma_event_channel *channel;
  63         struct cmatest_node     *nodes;
  64         int                     conn_index;
  65         int                     connects_left;
  66
  67         struct sockaddr_in      dst_in;
  68         struct sockaddr         *dst_addr;
  69         struct sockaddr_in      src_in;
  70         struct sockaddr         *src_addr;
  71 };
  72
  73 static struct cmatest test;
  74 static int connections = 1;
  75 static int message_size = 100;
  76 static int message_count = 10;
  77 static uint16_t port = 7174;
  78 static uint8_t set_tos = 0;
  79 static uint8_t tos;
  80 static char *dst_addr;
  81 static char *src_addr;
  82 static enum rdma_port_space port_space = RDMA_PS_UDP;
  83
  84 static int create_message(struct cmatest_node *node)
  85 {
  86         if (!message_size)
  87                 message_count = 0;
  88
  89         if (!message_count)
  90                 return 0;
  91
  92         node->mem = malloc(message_size + sizeof(struct ibv_grh));
  93         if (!node->mem) {
  94                 printf("failed message allocation\n");
  95                 return -1;
  96         }
  97         node->mr = ibv_reg_mr(node->pd, node->mem,
  98                               message_size + sizeof(struct ibv_grh),
  99                               IBV_ACCESS_LOCAL_WRITE);
 100         if (!node->mr) {
 101                 printf("failed to reg MR\n");
 102                 goto err;
 103         }
 104         return 0;
 105 err:
 106         free(node->mem);
 107         return -1;
 108 }
 109
 110 static int verify_test_params(struct cmatest_node *node)
 111 {
 112         struct ibv_port_attr port_attr;
 113         int ret;
 114
 115         ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num,
 116                              &port_attr);
 117         if (ret)
 118                 return ret;
 119
 120         if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) {
 121                 printf("udaddy: message_size %d is larger than active mtu %d\n",
 122                        message_size, 1 << (port_attr.active_mtu + 7));
 123                 return -EINVAL;
 124         }
 125
 126         return 0;
 127 }
 128
 129 static int init_node(struct cmatest_node *node)
 130 {
 131         struct ibv_qp_init_attr init_qp_attr;
 132         int cqe, ret;
 133
 134         node->pd = ibv_alloc_pd(node->cma_id->verbs);
 135         if (!node->pd) {
 136                 ret = -ENOMEM;
 137                 printf("udaddy: unable to allocate PD\n");
 138                 goto out;
 139         }
 140
 141         cqe = message_count ? message_count * 2 : 2;
 142         node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);
 143         if (!node->cq) {
 144                 ret = -ENOMEM;
 145                 printf("udaddy: unable to create CQ\n");
 146                 goto out;
 147         }
 148
 149         memset(&init_qp_attr, 0, sizeof init_qp_attr);
 150         init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;
 151         init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
 152         init_qp_attr.cap.max_send_sge = 1;
 153         init_qp_attr.cap.max_recv_sge = 1;
 154         init_qp_attr.qp_context = node;
 155         init_qp_attr.sq_sig_all = 0;
 156         init_qp_attr.qp_type = IBV_QPT_UD;
 157         init_qp_attr.send_cq = node->cq;
 158         init_qp_attr.recv_cq = node->cq;
 159         ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
 160         if (ret) {
 161                 perror("udaddy: unable to create QP");
 162                 goto out;
 163         }
 164
 165         ret = create_message(node);
 166         if (ret) {
 167                 printf("udaddy: failed to create messages: %d\n", ret);
 168                 goto out;
 169         }
 170 out:
 171         return ret;
 172 }
 173
 174 static int post_recvs(struct cmatest_node *node)
 175 {
 176         struct ibv_recv_wr recv_wr, *recv_failure;
 177         struct ibv_sge sge;
 178         int i, ret = 0;
 179
 180         if (!message_count)
 181                 return 0;
 182
 183         recv_wr.next = NULL;
 184         recv_wr.sg_list = &sge;
 185         recv_wr.num_sge = 1;
 186         recv_wr.wr_id = (uintptr_t) node;
 187
 188         sge.length = message_size + sizeof(struct ibv_grh);
 189         sge.lkey = node->mr->lkey;
 190         sge.addr = (uintptr_t) node->mem;
 191
 192         for (i = 0; i < message_count && !ret; i++ ) {
 193                 ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
 194                 if (ret) {
 195                         printf("failed to post receives: %d\n", ret);
 196                         break;
 197                 }
 198         }
 199         return ret;
 200 }
 201
 202 static int post_sends(struct cmatest_node *node, int signal_flag)
 203 {
 204         struct ibv_send_wr send_wr, *bad_send_wr;
 205         struct ibv_sge sge;
 206         int i, ret = 0;
 207
 208         if (!node->connected || !message_count)
 209                 return 0;
 210
 211         send_wr.next = NULL;
 212         send_wr.sg_list = &sge;
 213         send_wr.num_sge = 1;
 214         send_wr.opcode = IBV_WR_SEND_WITH_IMM;
 215         send_wr.send_flags = signal_flag;
 216         send_wr.wr_id = (unsigned long)node;
 217         send_wr.imm_data = htonl(node->cma_id->qp->qp_num);
 218
 219         send_wr.wr.ud.ah = node->ah;
 220         send_wr.wr.ud.remote_qpn = node->remote_qpn;
 221         send_wr.wr.ud.remote_qkey = node->remote_qkey;
 222
 223         sge.length = message_size;
 224         sge.lkey = node->mr->lkey;
 225         sge.addr = (uintptr_t) node->mem;
 226
 227         for (i = 0; i < message_count && !ret; i++) {
 228                 ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
 229                 if (ret)
 230                         printf("failed to post sends: %d\n", ret);
 231         }
 232         return ret;
 233 }
 234
 235 static void connect_error(void)
 236 {
 237         test.connects_left--;
 238 }
 239
 240 static int addr_handler(struct cmatest_node *node)
 241 {
 242         int ret;
 243
 244         if (set_tos) {
 245                 ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID,
 246                                       RDMA_OPTION_ID_TOS, &tos, sizeof tos);
 247                 if (ret)
 248                         perror("udaddy: set TOS option failed");
 249         }
 250
 251         ret = rdma_resolve_route(node->cma_id, 2000);
 252         if (ret) {
 253                 perror("udaddy: resolve route failed");
 254                 connect_error();
 255         }
 256         return ret;
 257 }
 258
 259 static int route_handler(struct cmatest_node *node)
 260 {
 261         struct rdma_conn_param conn_param;
 262         int ret;
 263
 264         ret = verify_test_params(node);
 265         if (ret)
 266                 goto err;
 267
 268         ret = init_node(node);
 269         if (ret)
 270                 goto err;
 271
 272         ret = post_recvs(node);
 273         if (ret)
 274                 goto err;
 275
 276         memset(&conn_param, 0, sizeof conn_param);
 277         ret = rdma_connect(node->cma_id, &conn_param);
 278         if (ret) {
 279                 perror("udaddy: failure connecting");
 280                 goto err;
 281         }
 282         return 0;
 283 err:
 284         connect_error();
 285         return ret;
 286 }
 287
 288 static int connect_handler(struct rdma_cm_id *cma_id)
 289 {
 290         struct cmatest_node *node;
 291         struct rdma_conn_param conn_param;
 292         int ret;
 293
 294         if (test.conn_index == connections) {
 295                 ret = -ENOMEM;
 296                 goto err1;
 297         }
 298         node = &test.nodes[test.conn_index++];
 299
 300         node->cma_id = cma_id;
 301         cma_id->context = node;
 302
 303         ret = verify_test_params(node);
 304         if (ret)
 305                 goto err2;
 306
 307         ret = init_node(node);
 308         if (ret)
 309                 goto err2;
 310
 311         ret = post_recvs(node);
 312         if (ret)
 313                 goto err2;
 314
 315         memset(&conn_param, 0, sizeof conn_param);
 316         conn_param.qp_num = node->cma_id->qp->qp_num;
 317         ret = rdma_accept(node->cma_id, &conn_param);
 318         if (ret) {
 319                 perror("udaddy: failure accepting");
 320                 goto err2;
 321         }
 322         node->connected = 1;
 323         test.connects_left--;
 324         return 0;
 325
 326 err2:
 327         node->cma_id = NULL;
 328         connect_error();
 329 err1:
 330         printf("udaddy: failing connection request\n");
 331         rdma_reject(cma_id, NULL, 0);
 332         return ret;
 333 }
 334
 335 static int resolved_handler(struct cmatest_node *node,
 336                             struct rdma_cm_event *event)
 337 {
 338         node->remote_qpn = event->param.ud.qp_num;
 339         node->remote_qkey = event->param.ud.qkey;
 340         node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr);
 341         if (!node->ah) {
 342                 printf("udaddy: failure creating address handle\n");
 343                 goto err;
 344         }
 345
 346         node->connected = 1;
 347         test.connects_left--;
 348         return 0;
 349 err:
 350         connect_error();
 351         return -1;
 352 }
 353
 354 static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 355 {
 356         int ret = 0;
 357
 358         switch (event->event) {
 359         case RDMA_CM_EVENT_ADDR_RESOLVED:
 360                 ret = addr_handler(cma_id->context);
 361                 break;
 362         case RDMA_CM_EVENT_ROUTE_RESOLVED:
 363                 ret = route_handler(cma_id->context);
 364                 break;
 365         case RDMA_CM_EVENT_CONNECT_REQUEST:
 366                 ret = connect_handler(cma_id);
 367                 break;
 368         case RDMA_CM_EVENT_ESTABLISHED:
 369                 ret = resolved_handler(cma_id->context, event);
 370                 break;
 371         case RDMA_CM_EVENT_ADDR_ERROR:
 372         case RDMA_CM_EVENT_ROUTE_ERROR:
 373         case RDMA_CM_EVENT_CONNECT_ERROR:
 374         case RDMA_CM_EVENT_UNREACHABLE:
 375         case RDMA_CM_EVENT_REJECTED:
 376                 printf("udaddy: event: %s, error: %d\n",
 377                        rdma_event_str(event->event), event->status);
 378                 connect_error();
 379                 ret = event->status;
 380                 break;
 381         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 382                 /* Cleanup will occur after test completes. */
 383                 break;
 384         default:
 385                 break;
 386         }
 387         return ret;
 388 }
 389
 390 static void destroy_node(struct cmatest_node *node)
 391 {
 392         if (!node->cma_id)
 393                 return;
 394
 395         if (node->ah)
 396                 ibv_destroy_ah(node->ah);
 397
 398         if (node->cma_id->qp)
 399                 rdma_destroy_qp(node->cma_id);
 400
 401         if (node->cq)
 402                 ibv_destroy_cq(node->cq);
 403
 404         if (node->mem) {
 405                 ibv_dereg_mr(node->mr);
 406                 free(node->mem);
 407         }
 408
 409         if (node->pd)
 410                 ibv_dealloc_pd(node->pd);
 411
 412         /* Destroy the RDMA ID after all device resources */
 413         rdma_destroy_id(node->cma_id);
 414 }
 415
 416 static int alloc_nodes(void)
 417 {
 418         int ret, i;
 419
 420         test.nodes = malloc(sizeof *test.nodes * connections);
 421         if (!test.nodes) {
 422                 printf("udaddy: unable to allocate memory for test nodes\n");
 423                 return -ENOMEM;
 424         }
 425         memset(test.nodes, 0, sizeof *test.nodes * connections);
 426
 427         for (i = 0; i < connections; i++) {
 428                 test.nodes[i].id = i;
 429                 if (dst_addr) {
 430                         ret = rdma_create_id(test.channel,
 431                                              &test.nodes[i].cma_id,
 432                                              &test.nodes[i], port_space);
 433                         if (ret)
 434                                 goto err;
 435                 }
 436         }
 437         return 0;
 438 err:
 439         while (--i >= 0)
 440                 rdma_destroy_id(test.nodes[i].cma_id);
 441         free(test.nodes);
 442         return ret;
 443 }
 444
 445 static void destroy_nodes(void)
 446 {
 447         int i;
 448
 449         for (i = 0; i < connections; i++)
 450                 destroy_node(&test.nodes[i]);
 451         free(test.nodes);
 452 }
 453
 454 static void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc)
 455 {
 456         struct ibv_qp_attr attr;
 457         struct ibv_qp_init_attr init_attr;
 458
 459         node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem,
 460                                          node->cma_id->port_num);
 461         node->remote_qpn = ntohl(wc->imm_data);
 462
 463         ibv_query_qp(node->cma_id->qp, &attr, IBV_QP_QKEY, &init_attr);
 464         node->remote_qkey = attr.qkey;
 465 }
 466
 467 static int poll_cqs(void)
 468 {
 469         struct ibv_wc wc[8];
 470         int done, i, ret;
 471
 472         for (i = 0; i < connections; i++) {
 473                 if (!test.nodes[i].connected)
 474                         continue;
 475
 476                 for (done = 0; done < message_count; done += ret) {
 477                         ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
 478                         if (ret < 0) {
 479                                 printf("udaddy: failed polling CQ: %d\n", ret);
 480                                 return ret;
 481                         }
 482
 483                         if (ret && !test.nodes[i].ah)
 484                                 create_reply_ah(&test.nodes[i], wc);
 485                 }
 486         }
 487         return 0;
 488 }
 489
 490 static int connect_events(void)
 491 {
 492         struct rdma_cm_event *event;
 493         int ret = 0;
 494
 495         while (test.connects_left && !ret) {
 496                 ret = rdma_get_cm_event(test.channel, &event);
 497                 if (!ret) {
 498                         ret = cma_handler(event->id, event);
 499                         rdma_ack_cm_event(event);
 500                 }
 501         }
 502         return ret;
 503 }
 504
 505 static int get_addr(char *dst, struct sockaddr_in *addr)
 506 {
 507         struct addrinfo *res;
 508         int ret;
 509
 510         ret = getaddrinfo(dst, NULL, NULL, &res);
 511         if (ret) {
 512                 printf("getaddrinfo failed - invalid hostname or IP address\n");
 513                 return ret;
 514         }
 515
 516         if (res->ai_family != PF_INET) {
 517                 ret = -1;
 518                 goto out;
 519         }
 520
 521         *addr = *(struct sockaddr_in *) res->ai_addr;
 522 out:
 523         freeaddrinfo(res);
 524         return ret;
 525 }
 526
 527 static int run_server(void)
 528 {
 529         struct rdma_cm_id *listen_id;
 530         int i, ret;
 531
 532         printf("udaddy: starting server\n");
 533         ret = rdma_create_id(test.channel, &listen_id, &test, port_space);
 534         if (ret) {
 535                 perror("udaddy: listen request failed");
 536                 return ret;
 537         }
 538
 539         if (src_addr) {
 540                 ret = get_addr(src_addr, &test.src_in);
 541                 if (ret)
 542                         goto out;
 543         } else
 544                 test.src_in.sin_family = PF_INET;
 545
 546         test.src_in.sin_port = port;
 547         ret = rdma_bind_addr(listen_id, test.src_addr);
 548         if (ret) {
 549                 perror("udaddy: bind address failed");
 550                 return ret;
 551         }
 552
 553         ret = rdma_listen(listen_id, 0);
 554         if (ret) {
 555                 perror("udaddy: failure trying to listen");
 556                 goto out;
 557         }
 558
 559         connect_events();
 560
 561         if (message_count) {
 562                 printf("receiving data transfers\n");
 563                 ret = poll_cqs();
 564                 if (ret)
 565                         goto out;
 566
 567                 printf("sending replies\n");
 568                 for (i = 0; i < connections; i++) {
 569                         ret = post_sends(&test.nodes[i], IBV_SEND_SIGNALED);
 570                         if (ret)
 571                                 goto out;
 572                 }
 573
 574                 ret = poll_cqs();
 575                 if (ret)
 576                         goto out;
 577                 printf("data transfers complete\n");
 578         }
 579 out:
 580         rdma_destroy_id(listen_id);
 581         return ret;
 582 }
 583
 584 static int run_client(void)
 585 {
 586         int i, ret;
 587
 588         printf("udaddy: starting client\n");
 589         if (src_addr) {
 590                 ret = get_addr(src_addr, &test.src_in);
 591                 if (ret)
 592                         return ret;
 593         }
 594
 595         ret = get_addr(dst_addr, &test.dst_in);
 596         if (ret)
 597                 return ret;
 598
 599         test.dst_in.sin_port = port;
 600
 601         printf("udaddy: connecting\n");
 602         for (i = 0; i < connections; i++) {
 603                 ret = rdma_resolve_addr(test.nodes[i].cma_id,
 604                                         src_addr ? test.src_addr : NULL,
 605                                         test.dst_addr, 2000);
 606                 if (ret) {
 607                         perror("udaddy: failure getting addr");
 608                         connect_error();
 609                         return ret;
 610                 }
 611         }
 612
 613         ret = connect_events();
 614         if (ret)
 615                 goto out;
 616
 617         if (message_count) {
 618                 printf("initiating data transfers\n");
 619                 for (i = 0; i < connections; i++) {
 620                         ret = post_sends(&test.nodes[i], 0);
 621                         if (ret)
 622                                 goto out;
 623                 }
 624                 printf("receiving data transfers\n");
 625                 ret = poll_cqs();
 626                 if (ret)
 627                         goto out;
 628
 629                 printf("data transfers complete\n");
 630         }
 631 out:
 632         return ret;
 633 }
 634
 635 int main(int argc, char **argv)
 636 {
 637         int op, ret;
 638
 639         while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:")) != -1) {
 640                 switch (op) {
 641                 case 's':
 642                         dst_addr = optarg;
 643                         break;
 644                 case 'b':
 645                         src_addr = optarg;
 646                         break;
 647                 case 'c':
 648                         connections = atoi(optarg);
 649                         break;
 650                 case 'C':
 651                         message_count = atoi(optarg);
 652                         break;
 653                 case 'S':
 654                         message_size = atoi(optarg);
 655                         break;
 656                 case 't':
 657                         set_tos = 1;
 658                         tos = (uint8_t) atoi(optarg);
 659                         break;
 660                 case 'p':
 661                         port_space = strtol(optarg, NULL, 0);
 662                         break;
 663                 default:
 664                         printf("usage: %s\n", argv[0]);
 665                         printf("\t[-s server_address]\n");
 666                         printf("\t[-b bind_address]\n");
 667                         printf("\t[-c connections]\n");
 668                         printf("\t[-C message_count]\n");
 669                         printf("\t[-S message_size]\n");
 670                         printf("\t[-t type_of_service]\n");
 671                         printf("\t[-p port_space - %#x for UDP (default), "
 672                                "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);
 673                         exit(1);
 674                 }
 675         }
 676
 677         test.dst_addr = (struct sockaddr *) &test.dst_in;
 678         test.src_addr = (struct sockaddr *) &test.src_in;
 679         test.connects_left = connections;
 680
 681         test.channel = rdma_create_event_channel();
 682         if (!test.channel) {
 683                 perror("failed to create event channel");
 684                 exit(1);
 685         }
 686
 687         if (alloc_nodes())
 688                 exit(1);
 689
 690         if (dst_addr)
 691                 ret = run_client();
 692         else
 693                 ret = run_server();
 694
 695         printf("test complete\n");
 696         destroy_nodes();
 697         rdma_destroy_event_channel(test.channel);
 698
 699         printf("return status %d\n", ret);
 700         return ret;
 701 }