contrib/ofed/librdmacm/examples/cmatose.c

   1 /*
   2  * Copyright (c) 2005-2006,2011-2012 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  * $Id$
  33  */
  34
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <stdio.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/socket.h>
  41 #include <netdb.h>
  42 #include <getopt.h>
  43
  44 #include <rdma/rdma_cma.h>
  45 #include "common.h"
  46
  47 struct cmatest_node {
  48         int                     id;
  49         struct rdma_cm_id       *cma_id;
  50         int                     connected;
  51         struct ibv_pd           *pd;
  52         struct ibv_cq           *cq[2];
  53         struct ibv_mr           *mr;
  54         void                    *mem;
  55 };
  56
  57 enum CQ_INDEX {
  58         SEND_CQ_INDEX,
  59         RECV_CQ_INDEX
  60 };
  61
  62 struct cmatest {
  63         struct rdma_event_channel *channel;
  64         struct cmatest_node     *nodes;
  65         int                     conn_index;
  66         int                     connects_left;
  67         int                     disconnects_left;
  68
  69         struct rdma_addrinfo    *rai;
  70 };
  71
  72 static struct cmatest test;
  73 static int connections = 1;
  74 static int message_size = 100;
  75 static int message_count = 10;
  76 static const char *port = "7471";
  77 static uint8_t set_tos = 0;
  78 static uint8_t tos;
  79 static uint8_t migrate = 0;
  80 static char *dst_addr;
  81 static char *src_addr;
  82 static struct rdma_addrinfo hints;
  83
  84 static int create_message(struct cmatest_node *node)
  85 {
  86         if (!message_size)
  87                 message_count = 0;
  88
  89         if (!message_count)
  90                 return 0;
  91
  92         node->mem = malloc(message_size);
  93         if (!node->mem) {
  94                 printf("failed message allocation\n");
  95                 return -1;
  96         }
  97         node->mr = ibv_reg_mr(node->pd, node->mem, message_size,
  98                              IBV_ACCESS_LOCAL_WRITE);
  99         if (!node->mr) {
 100                 printf("failed to reg MR\n");
 101                 goto err;
 102         }
 103         return 0;
 104 err:
 105         free(node->mem);
 106         return -1;
 107 }
 108
 109 static int init_node(struct cmatest_node *node)
 110 {
 111         struct ibv_qp_init_attr init_qp_attr;
 112         int cqe, ret;
 113
 114         node->pd = ibv_alloc_pd(node->cma_id->verbs);
 115         if (!node->pd) {
 116                 ret = -ENOMEM;
 117                 printf("cmatose: unable to allocate PD\n");
 118                 goto out;
 119         }
 120
 121         cqe = message_count ? message_count : 1;
 122         node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
 123         node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
 124         if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {
 125                 ret = -ENOMEM;
 126                 printf("cmatose: unable to create CQ\n");
 127                 goto out;
 128         }
 129
 130         memset(&init_qp_attr, 0, sizeof init_qp_attr);
 131         init_qp_attr.cap.max_send_wr = cqe;
 132         init_qp_attr.cap.max_recv_wr = cqe;
 133         init_qp_attr.cap.max_send_sge = 1;
 134         init_qp_attr.cap.max_recv_sge = 1;
 135         init_qp_attr.qp_context = node;
 136         init_qp_attr.sq_sig_all = 1;
 137         init_qp_attr.qp_type = IBV_QPT_RC;
 138         init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];
 139         init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];
 140         ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
 141         if (ret) {
 142                 perror("cmatose: unable to create QP");
 143                 goto out;
 144         }
 145
 146         ret = create_message(node);
 147         if (ret) {
 148                 printf("cmatose: failed to create messages: %d\n", ret);
 149                 goto out;
 150         }
 151 out:
 152         return ret;
 153 }
 154
 155 static int post_recvs(struct cmatest_node *node)
 156 {
 157         struct ibv_recv_wr recv_wr, *recv_failure;
 158         struct ibv_sge sge;
 159         int i, ret = 0;
 160
 161         if (!message_count)
 162                 return 0;
 163
 164         recv_wr.next = NULL;
 165         recv_wr.sg_list = &sge;
 166         recv_wr.num_sge = 1;
 167         recv_wr.wr_id = (uintptr_t) node;
 168
 169         sge.length = message_size;
 170         sge.lkey = node->mr->lkey;
 171         sge.addr = (uintptr_t) node->mem;
 172
 173         for (i = 0; i < message_count && !ret; i++ ) {
 174                 ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
 175                 if (ret) {
 176                         printf("failed to post receives: %d\n", ret);
 177                         break;
 178                 }
 179         }
 180         return ret;
 181 }
 182
 183 static int post_sends(struct cmatest_node *node)
 184 {
 185         struct ibv_send_wr send_wr, *bad_send_wr;
 186         struct ibv_sge sge;
 187         int i, ret = 0;
 188
 189         if (!node->connected || !message_count)
 190                 return 0;
 191
 192         send_wr.next = NULL;
 193         send_wr.sg_list = &sge;
 194         send_wr.num_sge = 1;
 195         send_wr.opcode = IBV_WR_SEND;
 196         send_wr.send_flags = 0;
 197         send_wr.wr_id = (unsigned long)node;
 198
 199         sge.length = message_size;
 200         sge.lkey = node->mr->lkey;
 201         sge.addr = (uintptr_t) node->mem;
 202
 203         for (i = 0; i < message_count && !ret; i++) {
 204                 ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
 205                 if (ret)
 206                         printf("failed to post sends: %d\n", ret);
 207         }
 208         return ret;
 209 }
 210
 211 static void connect_error(void)
 212 {
 213         test.connects_left--;
 214 }
 215
 216 static int addr_handler(struct cmatest_node *node)
 217 {
 218         int ret;
 219
 220         if (set_tos) {
 221                 ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID,
 222                                       RDMA_OPTION_ID_TOS, &tos, sizeof tos);
 223                 if (ret)
 224                         perror("cmatose: set TOS option failed");
 225         }
 226
 227         ret = rdma_resolve_route(node->cma_id, 2000);
 228         if (ret) {
 229                 perror("cmatose: resolve route failed");
 230                 connect_error();
 231         }
 232         return ret;
 233 }
 234
 235 static int route_handler(struct cmatest_node *node)
 236 {
 237         struct rdma_conn_param conn_param;
 238         int ret;
 239
 240         ret = init_node(node);
 241         if (ret)
 242                 goto err;
 243
 244         ret = post_recvs(node);
 245         if (ret)
 246                 goto err;
 247
 248         memset(&conn_param, 0, sizeof conn_param);
 249         conn_param.responder_resources = 1;
 250         conn_param.initiator_depth = 1;
 251         conn_param.retry_count = 5;
 252         conn_param.private_data = test.rai->ai_connect;
 253         conn_param.private_data_len = test.rai->ai_connect_len;
 254         ret = rdma_connect(node->cma_id, &conn_param);
 255         if (ret) {
 256                 perror("cmatose: failure connecting");
 257                 goto err;
 258         }
 259         return 0;
 260 err:
 261         connect_error();
 262         return ret;
 263 }
 264
 265 static int connect_handler(struct rdma_cm_id *cma_id)
 266 {
 267         struct cmatest_node *node;
 268         int ret;
 269
 270         if (test.conn_index == connections) {
 271                 ret = -ENOMEM;
 272                 goto err1;
 273         }
 274         node = &test.nodes[test.conn_index++];
 275
 276         node->cma_id = cma_id;
 277         cma_id->context = node;
 278
 279         ret = init_node(node);
 280         if (ret)
 281                 goto err2;
 282
 283         ret = post_recvs(node);
 284         if (ret)
 285                 goto err2;
 286
 287         ret = rdma_accept(node->cma_id, NULL);
 288         if (ret) {
 289                 perror("cmatose: failure accepting");
 290                 goto err2;
 291         }
 292         return 0;
 293
 294 err2:
 295         node->cma_id = NULL;
 296         connect_error();
 297 err1:
 298         printf("cmatose: failing connection request\n");
 299         rdma_reject(cma_id, NULL, 0);
 300         return ret;
 301 }
 302
 303 static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 304 {
 305         int ret = 0;
 306
 307         switch (event->event) {
 308         case RDMA_CM_EVENT_ADDR_RESOLVED:
 309                 ret = addr_handler(cma_id->context);
 310                 break;
 311         case RDMA_CM_EVENT_ROUTE_RESOLVED:
 312                 ret = route_handler(cma_id->context);
 313                 break;
 314         case RDMA_CM_EVENT_CONNECT_REQUEST:
 315                 ret = connect_handler(cma_id);
 316                 break;
 317         case RDMA_CM_EVENT_ESTABLISHED:
 318                 ((struct cmatest_node *) cma_id->context)->connected = 1;
 319                 test.connects_left--;
 320                 test.disconnects_left++;
 321                 break;
 322         case RDMA_CM_EVENT_ADDR_ERROR:
 323         case RDMA_CM_EVENT_ROUTE_ERROR:
 324         case RDMA_CM_EVENT_CONNECT_ERROR:
 325         case RDMA_CM_EVENT_UNREACHABLE:
 326         case RDMA_CM_EVENT_REJECTED:
 327                 printf("cmatose: event: %s, error: %d\n",
 328                        rdma_event_str(event->event), event->status);
 329                 connect_error();
 330                 ret = event->status;
 331                 break;
 332         case RDMA_CM_EVENT_DISCONNECTED:
 333                 rdma_disconnect(cma_id);
 334                 test.disconnects_left--;
 335                 break;
 336         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 337                 /* Cleanup will occur after test completes. */
 338                 break;
 339         default:
 340                 break;
 341         }
 342         return ret;
 343 }
 344
 345 static void destroy_node(struct cmatest_node *node)
 346 {
 347         if (!node->cma_id)
 348                 return;
 349
 350         if (node->cma_id->qp)
 351                 rdma_destroy_qp(node->cma_id);
 352
 353         if (node->cq[SEND_CQ_INDEX])
 354                 ibv_destroy_cq(node->cq[SEND_CQ_INDEX]);
 355
 356         if (node->cq[RECV_CQ_INDEX])
 357                 ibv_destroy_cq(node->cq[RECV_CQ_INDEX]);
 358
 359         if (node->mem) {
 360                 ibv_dereg_mr(node->mr);
 361                 free(node->mem);
 362         }
 363
 364         if (node->pd)
 365                 ibv_dealloc_pd(node->pd);
 366
 367         /* Destroy the RDMA ID after all device resources */
 368         rdma_destroy_id(node->cma_id);
 369 }
 370
 371 static int alloc_nodes(void)
 372 {
 373         int ret, i;
 374
 375         test.nodes = malloc(sizeof *test.nodes * connections);
 376         if (!test.nodes) {
 377                 printf("cmatose: unable to allocate memory for test nodes\n");
 378                 return -ENOMEM;
 379         }
 380         memset(test.nodes, 0, sizeof *test.nodes * connections);
 381
 382         for (i = 0; i < connections; i++) {
 383                 test.nodes[i].id = i;
 384                 if (dst_addr) {
 385                         ret = rdma_create_id(test.channel,
 386                                              &test.nodes[i].cma_id,
 387                                              &test.nodes[i], hints.ai_port_space);
 388                         if (ret)
 389                                 goto err;
 390                 }
 391         }
 392         return 0;
 393 err:
 394         while (--i >= 0)
 395                 rdma_destroy_id(test.nodes[i].cma_id);
 396         free(test.nodes);
 397         return ret;
 398 }
 399
 400 static void destroy_nodes(void)
 401 {
 402         int i;
 403
 404         for (i = 0; i < connections; i++)
 405                 destroy_node(&test.nodes[i]);
 406         free(test.nodes);
 407 }
 408
 409 static int poll_cqs(enum CQ_INDEX index)
 410 {
 411         struct ibv_wc wc[8];
 412         int done, i, ret;
 413
 414         for (i = 0; i < connections; i++) {
 415                 if (!test.nodes[i].connected)
 416                         continue;
 417
 418                 for (done = 0; done < message_count; done += ret) {
 419                         ret = ibv_poll_cq(test.nodes[i].cq[index], 8, wc);
 420                         if (ret < 0) {
 421                                 printf("cmatose: failed polling CQ: %d\n", ret);
 422                                 return ret;
 423                         }
 424                 }
 425         }
 426         return 0;
 427 }
 428
 429 static int connect_events(void)
 430 {
 431         struct rdma_cm_event *event;
 432         int ret = 0;
 433
 434         while (test.connects_left && !ret) {
 435                 ret = rdma_get_cm_event(test.channel, &event);
 436                 if (!ret) {
 437                         ret = cma_handler(event->id, event);
 438                         rdma_ack_cm_event(event);
 439                 } else {
 440                         perror("cmatose: failure in rdma_get_cm_event in connect events");
 441                         ret = errno;
 442                 }
 443         }
 444
 445         return ret;
 446 }
 447
 448 static int disconnect_events(void)
 449 {
 450         struct rdma_cm_event *event;
 451         int ret = 0;
 452
 453         while (test.disconnects_left && !ret) {
 454                 ret = rdma_get_cm_event(test.channel, &event);
 455                 if (!ret) {
 456                         ret = cma_handler(event->id, event);
 457                         rdma_ack_cm_event(event);
 458                 } else {
 459                         perror("cmatose: failure in rdma_get_cm_event in disconnect events");
 460                         ret = errno;
 461                 }
 462         }
 463
 464         return ret;
 465 }
 466
 467 static int migrate_channel(struct rdma_cm_id *listen_id)
 468 {
 469         struct rdma_event_channel *channel;
 470         int i, ret;
 471
 472         printf("migrating to new event channel\n");
 473
 474         channel = rdma_create_event_channel();
 475         if (!channel) {
 476                 perror("cmatose: failed to create event channel");
 477                 return -1;
 478         }
 479
 480         ret = 0;
 481         if (listen_id)
 482                 ret = rdma_migrate_id(listen_id, channel);
 483
 484         for (i = 0; i < connections && !ret; i++)
 485                 ret = rdma_migrate_id(test.nodes[i].cma_id, channel);
 486
 487         if (!ret) {
 488                 rdma_destroy_event_channel(test.channel);
 489                 test.channel = channel;
 490         } else
 491                 perror("cmatose: failure migrating to channel");
 492
 493         return ret;
 494 }
 495
 496 static int run_server(void)
 497 {
 498         struct rdma_cm_id *listen_id;
 499         int i, ret;
 500
 501         printf("cmatose: starting server\n");
 502         ret = rdma_create_id(test.channel, &listen_id, &test, hints.ai_port_space);
 503         if (ret) {
 504                 perror("cmatose: listen request failed");
 505                 return ret;
 506         }
 507
 508         ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
 509         if (ret) {
 510                 printf("cmatose: getrdmaaddr error: %s\n", gai_strerror(ret));
 511                 goto out;
 512         }
 513
 514         ret = rdma_bind_addr(listen_id, test.rai->ai_src_addr);
 515         if (ret) {
 516                 perror("cmatose: bind address failed");
 517                 goto out;
 518         }
 519
 520         ret = rdma_listen(listen_id, 0);
 521         if (ret) {
 522                 perror("cmatose: failure trying to listen");
 523                 goto out;
 524         }
 525
 526         ret = connect_events();
 527         if (ret)
 528                 goto out;
 529
 530         if (message_count) {
 531                 printf("initiating data transfers\n");
 532                 for (i = 0; i < connections; i++) {
 533                         ret = post_sends(&test.nodes[i]);
 534                         if (ret)
 535                                 goto out;
 536                 }
 537
 538                 printf("completing sends\n");
 539                 ret = poll_cqs(SEND_CQ_INDEX);
 540                 if (ret)
 541                         goto out;
 542
 543                 printf("receiving data transfers\n");
 544                 ret = poll_cqs(RECV_CQ_INDEX);
 545                 if (ret)
 546                         goto out;
 547                 printf("data transfers complete\n");
 548
 549         }
 550
 551         if (migrate) {
 552                 ret = migrate_channel(listen_id);
 553                 if (ret)
 554                         goto out;
 555         }
 556
 557         printf("cmatose: disconnecting\n");
 558         for (i = 0; i < connections; i++) {
 559                 if (!test.nodes[i].connected)
 560                         continue;
 561
 562                 test.nodes[i].connected = 0;
 563                 rdma_disconnect(test.nodes[i].cma_id);
 564         }
 565
 566         ret = disconnect_events();
 567
 568         printf("disconnected\n");
 569
 570 out:
 571         rdma_destroy_id(listen_id);
 572         return ret;
 573 }
 574
 575 static int run_client(void)
 576 {
 577         int i, ret, ret2;
 578
 579         printf("cmatose: starting client\n");
 580
 581         ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
 582         if (ret) {
 583                 printf("cmatose: getaddrinfo error: %s\n", gai_strerror(ret));
 584                 return ret;
 585         }
 586
 587         printf("cmatose: connecting\n");
 588         for (i = 0; i < connections; i++) {
 589                 ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr,
 590                                         test.rai->ai_dst_addr, 2000);
 591                 if (ret) {
 592                         perror("cmatose: failure getting addr");
 593                         connect_error();
 594                         return ret;
 595                 }
 596         }
 597
 598         ret = connect_events();
 599         if (ret)
 600                 goto disc;
 601
 602         if (message_count) {
 603                 printf("receiving data transfers\n");
 604                 ret = poll_cqs(RECV_CQ_INDEX);
 605                 if (ret)
 606                         goto disc;
 607
 608                 printf("sending replies\n");
 609                 for (i = 0; i < connections; i++) {
 610                         ret = post_sends(&test.nodes[i]);
 611                         if (ret)
 612                                 goto disc;
 613                 }
 614
 615                 printf("data transfers complete\n");
 616         }
 617
 618         ret = 0;
 619
 620         if (migrate) {
 621                 ret = migrate_channel(NULL);
 622                 if (ret)
 623                         goto out;
 624         }
 625 disc:
 626         ret2 = disconnect_events();
 627         if (ret2)
 628                 ret = ret2;
 629 out:
 630         return ret;
 631 }
 632
 633 int main(int argc, char **argv)
 634 {
 635         int op, ret;
 636
 637         hints.ai_port_space = RDMA_PS_TCP;
 638         while ((op = getopt(argc, argv, "s:b:f:P:c:C:S:t:p:m")) != -1) {
 639                 switch (op) {
 640                 case 's':
 641                         dst_addr = optarg;
 642                         break;
 643                 case 'b':
 644                         src_addr = optarg;
 645                         break;
 646                 case 'f':
 647                         if (!strncasecmp("ip", optarg, 2)) {
 648                                 hints.ai_flags = RAI_NUMERICHOST;
 649                         } else if (!strncasecmp("gid", optarg, 3)) {
 650                                 hints.ai_flags = RAI_NUMERICHOST | RAI_FAMILY;
 651                                 hints.ai_family = AF_IB;
 652                         } else if (strncasecmp("name", optarg, 4)) {
 653                                 fprintf(stderr, "Warning: unknown address format\n");
 654                         }
 655                         break;
 656                 case 'P':
 657                         if (!strncasecmp("ib", optarg, 2)) {
 658                                 hints.ai_port_space = RDMA_PS_IB;
 659                         } else if (strncasecmp("tcp", optarg, 3)) {
 660                                 fprintf(stderr, "Warning: unknown port space format\n");
 661                         }
 662                         break;
 663                 case 'c':
 664                         connections = atoi(optarg);
 665                         break;
 666                 case 'C':
 667                         message_count = atoi(optarg);
 668                         break;
 669                 case 'S':
 670                         message_size = atoi(optarg);
 671                         break;
 672                 case 't':
 673                         set_tos = 1;
 674                         tos = (uint8_t) strtoul(optarg, NULL, 0);
 675                         break;
 676                 case 'p':
 677                         port = optarg;
 678                         break;
 679                 case 'm':
 680                         migrate = 1;
 681                         break;
 682                 default:
 683                         printf("usage: %s\n", argv[0]);
 684                         printf("\t[-s server_address]\n");
 685                         printf("\t[-b bind_address]\n");
 686                         printf("\t[-f address_format]\n");
 687                         printf("\t    name, ip, ipv6, or gid\n");
 688                         printf("\t[-P port_space]\n");
 689                         printf("\t    tcp or ib\n");
 690                         printf("\t[-c connections]\n");
 691                         printf("\t[-C message_count]\n");
 692                         printf("\t[-S message_size]\n");
 693                         printf("\t[-t type_of_service]\n");
 694                         printf("\t[-p port_number]\n");
 695                         printf("\t[-m(igrate)]\n");
 696                         exit(1);
 697                 }
 698         }
 699
 700         test.connects_left = connections;
 701
 702         test.channel = rdma_create_event_channel();
 703         if (!test.channel) {
 704                 printf("failed to create event channel\n");
 705                 exit(1);
 706         }
 707
 708         if (alloc_nodes())
 709                 exit(1);
 710
 711         if (dst_addr) {
 712                 ret = run_client();
 713         } else {
 714                 hints.ai_flags |= RAI_PASSIVE;
 715                 ret = run_server();
 716         }
 717
 718         printf("test complete\n");
 719         destroy_nodes();
 720         rdma_destroy_event_channel(test.channel);
 721         if (test.rai)
 722                 rdma_freeaddrinfo(test.rai);
 723
 724         printf("return status %d\n", ret);
 725         return ret;
 726 }