2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2011 Intel Corporation, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
42 #include <sys/types.h>
43 #include <sys/socket.h>
48 #include <arpa/inet.h>
53 #define MSG_FORMAT "%04x:%06x:%06x:%06x:%06x:%32s"
55 #define MSG_SSCAN "%x:%x:%x:%x:%x:%s"
57 "%8s: LID %04x, QPN RECV %06x SEND %06x, PSN %06x, SRQN %06x, GID %s\n"
58 #define TERMINATION_FORMAT "%s"
59 #define TERMINATION_MSG_SIZE 4
60 #define TERMINATION_MSG "END"
63 struct pingpong_dest {
75 struct pingpong_context {
76 struct ibv_context *context;
77 struct ibv_comp_channel *channel;
80 struct ibv_cq *send_cq;
81 struct ibv_cq *recv_cq;
83 struct ibv_xrcd *xrcd;
84 struct ibv_qp **recv_qp;
85 struct ibv_qp **send_qp;
86 struct pingpong_dest *rem_dest;
100 static struct pingpong_context ctx;
103 static int open_device(char *ib_devname)
105 struct ibv_device **dev_list;
108 dev_list = ibv_get_device_list(NULL);
110 fprintf(stderr, "Failed to get IB devices list");
115 for (; dev_list[i]; ++i) {
116 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
121 fprintf(stderr, "IB device %s not found\n",
122 ib_devname ? ib_devname : "");
126 ctx.context = ibv_open_device(dev_list[i]);
128 fprintf(stderr, "Couldn't get context for %s\n",
129 ibv_get_device_name(dev_list[i]));
133 ibv_free_device_list(dev_list);
137 static int create_qps(void)
139 struct ibv_qp_init_attr_ex init;
140 struct ibv_qp_attr mod;
143 for (i = 0; i < ctx.num_clients; ++i) {
145 memset(&init, 0, sizeof init);
146 init.qp_type = IBV_QPT_XRC_RECV;
147 init.comp_mask = IBV_QP_INIT_ATTR_XRCD;
148 init.xrcd = ctx.xrcd;
150 ctx.recv_qp[i] = ibv_create_qp_ex(ctx.context, &init);
151 if (!ctx.recv_qp[i]) {
152 fprintf(stderr, "Couldn't create recv QP[%d] errno %d\n",
157 mod.qp_state = IBV_QPS_INIT;
159 mod.port_num = ctx.ib_port;
160 mod.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
162 if (ibv_modify_qp(ctx.recv_qp[i], &mod,
163 IBV_QP_STATE | IBV_QP_PKEY_INDEX |
164 IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
165 fprintf(stderr, "Failed to modify recv QP[%d] to INIT\n", i);
169 memset(&init, 0, sizeof init);
170 init.qp_type = IBV_QPT_XRC_SEND;
171 init.send_cq = ctx.send_cq;
172 init.cap.max_send_wr = ctx.num_clients * ctx.num_tests;
173 init.cap.max_send_sge = 1;
174 init.comp_mask = IBV_QP_INIT_ATTR_PD;
177 ctx.send_qp[i] = ibv_create_qp_ex(ctx.context, &init);
178 if (!ctx.send_qp[i]) {
179 fprintf(stderr, "Couldn't create send QP[%d] errno %d\n",
184 mod.qp_state = IBV_QPS_INIT;
186 mod.port_num = ctx.ib_port;
187 mod.qp_access_flags = 0;
189 if (ibv_modify_qp(ctx.send_qp[i], &mod,
190 IBV_QP_STATE | IBV_QP_PKEY_INDEX |
191 IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
192 fprintf(stderr, "Failed to modify send QP[%d] to INIT\n", i);
200 static int pp_init_ctx(char *ib_devname)
202 struct ibv_srq_init_attr_ex attr;
203 struct ibv_xrcd_init_attr xrcd_attr;
204 struct ibv_port_attr port_attr;
206 ctx.recv_qp = calloc(ctx.num_clients, sizeof *ctx.recv_qp);
207 ctx.send_qp = calloc(ctx.num_clients, sizeof *ctx.send_qp);
208 ctx.rem_dest = calloc(ctx.num_clients, sizeof *ctx.rem_dest);
209 if (!ctx.recv_qp || !ctx.send_qp || !ctx.rem_dest)
212 if (open_device(ib_devname)) {
213 fprintf(stderr, "Failed to open device\n");
217 if (pp_get_port_info(ctx.context, ctx.ib_port, &port_attr)) {
218 fprintf(stderr, "Failed to get port info\n");
222 ctx.lid = port_attr.lid;
223 if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET && !ctx.lid) {
224 fprintf(stderr, "Couldn't get local LID\n");
228 ctx.buf = memalign(page_size, ctx.size);
230 fprintf(stderr, "Couldn't allocate work buf.\n");
234 memset(ctx.buf, 0, ctx.size);
237 ctx.channel = ibv_create_comp_channel(ctx.context);
239 fprintf(stderr, "Couldn't create completion channel\n");
244 ctx.pd = ibv_alloc_pd(ctx.context);
246 fprintf(stderr, "Couldn't allocate PD\n");
250 ctx.mr = ibv_reg_mr(ctx.pd, ctx.buf, ctx.size, IBV_ACCESS_LOCAL_WRITE);
252 fprintf(stderr, "Couldn't register MR\n");
256 ctx.fd = open("/tmp/xrc_domain", O_RDONLY | O_CREAT, S_IRUSR | S_IRGRP);
259 "Couldn't create the file for the XRC Domain "
260 "but not stopping %d\n", errno);
264 memset(&xrcd_attr, 0, sizeof xrcd_attr);
265 xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS;
266 xrcd_attr.fd = ctx.fd;
267 xrcd_attr.oflags = O_CREAT;
268 ctx.xrcd = ibv_open_xrcd(ctx.context, &xrcd_attr);
270 fprintf(stderr, "Couldn't Open the XRC Domain %d\n", errno);
274 ctx.recv_cq = ibv_create_cq(ctx.context, ctx.num_clients, &ctx.recv_cq,
277 fprintf(stderr, "Couldn't create recv CQ\n");
282 if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
283 fprintf(stderr, "Couldn't request CQ notification\n");
288 ctx.send_cq = ibv_create_cq(ctx.context, ctx.num_clients, NULL, NULL, 0);
290 fprintf(stderr, "Couldn't create send CQ\n");
294 memset(&attr, 0, sizeof attr);
295 attr.attr.max_wr = ctx.num_clients;
296 attr.attr.max_sge = 1;
297 attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_XRCD |
298 IBV_SRQ_INIT_ATTR_CQ | IBV_SRQ_INIT_ATTR_PD;
299 attr.srq_type = IBV_SRQT_XRC;
300 attr.xrcd = ctx.xrcd;
301 attr.cq = ctx.recv_cq;
304 ctx.srq = ibv_create_srq_ex(ctx.context, &attr);
306 fprintf(stderr, "Couldn't create SRQ\n");
316 static int recv_termination_ack(int index)
318 char msg[TERMINATION_MSG_SIZE];
320 int sockfd = ctx.rem_dest[index].sockfd;
322 while (n < TERMINATION_MSG_SIZE) {
323 r = read(sockfd, msg + n, TERMINATION_MSG_SIZE - n);
325 perror("client read");
327 "%d/%d: Couldn't read remote termination ack\n",
328 n, TERMINATION_MSG_SIZE);
334 if (strcmp(msg, TERMINATION_MSG)) {
335 fprintf(stderr, "Invalid termination ack was accepted\n");
342 static int send_termination_ack(int index)
344 char msg[TERMINATION_MSG_SIZE];
345 int sockfd = ctx.rem_dest[index].sockfd;
347 sprintf(msg, TERMINATION_FORMAT, TERMINATION_MSG);
349 if (write(sockfd, msg, TERMINATION_MSG_SIZE) != TERMINATION_MSG_SIZE) {
350 fprintf(stderr, "Couldn't send termination ack\n");
357 static int pp_client_termination(void)
359 if (send_termination_ack(0))
361 if (recv_termination_ack(0))
367 static int pp_server_termination(void)
371 for (i = 0; i < ctx.num_clients; i++) {
372 if (recv_termination_ack(i))
376 for (i = 0; i < ctx.num_clients; i++) {
377 if (send_termination_ack(i))
384 static int send_local_dest(int sockfd, int index)
389 union ibv_gid local_gid;
392 if (ibv_query_gid(ctx.context, ctx.ib_port, ctx.gidx,
394 fprintf(stderr, "can't read sgid of index %d\n",
399 memset(&local_gid, 0, sizeof(local_gid));
402 ctx.rem_dest[index].recv_psn = lrand48() & 0xffffff;
403 if (ibv_get_srq_num(ctx.srq, &srq_num)) {
404 fprintf(stderr, "Couldn't get SRQ num\n");
408 inet_ntop(AF_INET6, &local_gid, gid, sizeof(gid));
409 printf(ADDR_FORMAT, "local", ctx.lid, ctx.recv_qp[index]->qp_num,
410 ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
413 gid_to_wire_gid(&local_gid, gid);
414 sprintf(msg, MSG_FORMAT, ctx.lid, ctx.recv_qp[index]->qp_num,
415 ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
418 if (write(sockfd, msg, MSG_SIZE) != MSG_SIZE) {
419 fprintf(stderr, "Couldn't send local address\n");
426 static int recv_remote_dest(int sockfd, int index)
428 struct pingpong_dest *rem_dest;
433 while (n < MSG_SIZE) {
434 r = read(sockfd, msg + n, MSG_SIZE - n);
436 perror("client read");
438 "%d/%d: Couldn't read remote address [%d]\n",
445 rem_dest = &ctx.rem_dest[index];
446 sscanf(msg, MSG_SSCAN, &rem_dest->lid, &rem_dest->recv_qpn,
447 &rem_dest->send_qpn, &rem_dest->send_psn, &rem_dest->srqn, gid);
449 wire_gid_to_gid(gid, &rem_dest->gid);
450 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof(gid));
451 printf(ADDR_FORMAT, "remote", rem_dest->lid, rem_dest->recv_qpn,
452 rem_dest->send_qpn, rem_dest->send_psn, rem_dest->srqn,
455 rem_dest->sockfd = sockfd;
459 static void set_ah_attr(struct ibv_ah_attr *attr, struct pingpong_context *myctx,
463 attr->grh.hop_limit = 5;
464 attr->grh.dgid = myctx->rem_dest[index].gid;
465 attr->grh.sgid_index = myctx->gidx;
468 static int connect_qps(int index)
470 struct ibv_qp_attr attr;
472 memset(&attr, 0, sizeof attr);
473 attr.qp_state = IBV_QPS_RTR;
474 attr.dest_qp_num = ctx.rem_dest[index].send_qpn;
475 attr.path_mtu = ctx.mtu;
476 attr.rq_psn = ctx.rem_dest[index].send_psn;
477 attr.min_rnr_timer = 12;
478 attr.ah_attr.dlid = ctx.rem_dest[index].lid;
479 attr.ah_attr.sl = ctx.sl;
480 attr.ah_attr.port_num = ctx.ib_port;
482 if (ctx.rem_dest[index].gid.global.interface_id)
483 set_ah_attr(&attr.ah_attr, &ctx, index);
485 if (ibv_modify_qp(ctx.recv_qp[index], &attr,
486 IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
487 IBV_QP_DEST_QPN | IBV_QP_RQ_PSN |
488 IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) {
489 fprintf(stderr, "Failed to modify recv QP[%d] to RTR\n", index);
493 memset(&attr, 0, sizeof attr);
494 attr.qp_state = IBV_QPS_RTS;
496 attr.sq_psn = ctx.rem_dest[index].recv_psn;
498 if (ibv_modify_qp(ctx.recv_qp[index], &attr,
499 IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN)) {
500 fprintf(stderr, "Failed to modify recv QP[%d] to RTS\n", index);
504 memset(&attr, 0, sizeof attr);
505 attr.qp_state = IBV_QPS_RTR;
506 attr.dest_qp_num = ctx.rem_dest[index].recv_qpn;
507 attr.path_mtu = ctx.mtu;
508 attr.rq_psn = ctx.rem_dest[index].send_psn;
509 attr.ah_attr.dlid = ctx.rem_dest[index].lid;
510 attr.ah_attr.sl = ctx.sl;
511 attr.ah_attr.port_num = ctx.ib_port;
513 if (ctx.rem_dest[index].gid.global.interface_id)
514 set_ah_attr(&attr.ah_attr, &ctx, index);
516 if (ibv_modify_qp(ctx.send_qp[index], &attr,
517 IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
518 IBV_QP_DEST_QPN | IBV_QP_RQ_PSN)) {
519 fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);
523 memset(&attr, 0, sizeof attr);
524 attr.qp_state = IBV_QPS_RTS;
528 attr.sq_psn = ctx.rem_dest[index].recv_psn;
530 if (ibv_modify_qp(ctx.send_qp[index], &attr,
531 IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN |
532 IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) {
533 fprintf(stderr, "Failed to modify send QP[%d] to RTS\n", index);
540 static int pp_client_connect(const char *servername, int port)
542 struct addrinfo *res, *t;
546 struct addrinfo hints = {
547 .ai_family = AF_UNSPEC,
548 .ai_socktype = SOCK_STREAM
551 if (asprintf(&service, "%d", port) < 0)
554 ret = getaddrinfo(servername, service, &hints, &res);
556 fprintf(stderr, "%s for %s:%d\n", gai_strerror(ret), servername, port);
561 for (t = res; t; t = t->ai_next) {
562 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
564 if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
571 freeaddrinfo_null(res);
575 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
579 if (send_local_dest(sockfd, 0))
582 if (recv_remote_dest(sockfd, 0))
591 static int pp_server_connect(int port)
593 struct addrinfo *res, *t;
596 int sockfd = -1, connfd;
597 struct addrinfo hints = {
598 .ai_flags = AI_PASSIVE,
599 .ai_family = AF_INET,
600 .ai_socktype = SOCK_STREAM
603 if (asprintf(&service, "%d", port) < 0)
606 ret = getaddrinfo(NULL, service, &hints, &res);
608 fprintf(stderr, "%s for port %d\n", gai_strerror(ret), port);
613 for (t = res; t; t = t->ai_next) {
614 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
617 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
618 if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
625 freeaddrinfo_null(res);
629 fprintf(stderr, "Couldn't listen to port %d\n", port);
633 listen(sockfd, ctx.num_clients);
635 for (i = 0; i < ctx.num_clients; i++) {
636 connfd = accept(sockfd, NULL, NULL);
638 fprintf(stderr, "accept() failed for client %d\n", i);
642 if (recv_remote_dest(connfd, i))
645 if (send_local_dest(connfd, i))
657 static int pp_close_ctx(void)
661 for (i = 0; i < ctx.num_clients; ++i) {
663 if (ibv_destroy_qp(ctx.send_qp[i])) {
664 fprintf(stderr, "Couldn't destroy INI QP[%d]\n", i);
668 if (ibv_destroy_qp(ctx.recv_qp[i])) {
669 fprintf(stderr, "Couldn't destroy TGT QP[%d]\n", i);
673 if (ctx.rem_dest[i].sockfd)
674 close(ctx.rem_dest[i].sockfd);
677 if (ibv_destroy_srq(ctx.srq)) {
678 fprintf(stderr, "Couldn't destroy SRQ\n");
682 if (ctx.xrcd && ibv_close_xrcd(ctx.xrcd)) {
683 fprintf(stderr, "Couldn't close the XRC Domain\n");
686 if (ctx.fd >= 0 && close(ctx.fd)) {
687 fprintf(stderr, "Couldn't close the file for the XRC Domain\n");
691 if (ibv_destroy_cq(ctx.send_cq)) {
692 fprintf(stderr, "Couldn't destroy send CQ\n");
696 if (ibv_destroy_cq(ctx.recv_cq)) {
697 fprintf(stderr, "Couldn't destroy recv CQ\n");
701 if (ibv_dereg_mr(ctx.mr)) {
702 fprintf(stderr, "Couldn't deregister MR\n");
706 if (ibv_dealloc_pd(ctx.pd)) {
707 fprintf(stderr, "Couldn't deallocate PD\n");
712 if (ibv_destroy_comp_channel(ctx.channel)) {
714 "Couldn't destroy completion channel\n");
719 if (ibv_close_device(ctx.context)) {
720 fprintf(stderr, "Couldn't release context\n");
731 static int pp_post_recv(int cnt)
734 struct ibv_recv_wr wr, *bad_wr;
736 sge.addr = (uintptr_t) ctx.buf;
737 sge.length = ctx.size;
738 sge.lkey = ctx.mr->lkey;
741 wr.wr_id = (uintptr_t) &ctx;
746 if (ibv_post_srq_recv(ctx.srq, &wr, &bad_wr)) {
747 fprintf(stderr, "Failed to post receive to SRQ\n");
755 * Send to each client round robin on each set of xrc send/recv qp.
756 * Generate a completion on the last send.
758 static int pp_post_send(int index)
761 struct ibv_send_wr wr, *bad_wr;
764 sge.addr = (uintptr_t) ctx.buf;
765 sge.length = ctx.size;
766 sge.lkey = ctx.mr->lkey;
768 wr.wr_id = (uintptr_t) index;
772 wr.opcode = IBV_WR_SEND;
773 wr.qp_type.xrc.remote_srqn = ctx.rem_dest[index].srqn;
775 qpi = (index + ctx.rem_dest[index].pp_cnt) % ctx.num_clients;
776 wr.send_flags = (++ctx.rem_dest[index].pp_cnt >= ctx.num_tests) ?
777 IBV_SEND_SIGNALED : 0;
779 return ibv_post_send(ctx.send_qp[qpi], &wr, &bad_wr);
782 static int find_qp(int qpn)
786 if (ctx.num_clients == 1)
789 for (i = 0; i < ctx.num_clients; ++i)
790 if (ctx.recv_qp[i]->qp_num == qpn)
793 fprintf(stderr, "Unable to find qp %x\n", qpn);
797 static int get_cq_event(void)
799 struct ibv_cq *ev_cq;
802 if (ibv_get_cq_event(ctx.channel, &ev_cq, &ev_ctx)) {
803 fprintf(stderr, "Failed to get cq_event\n");
807 if (ev_cq != ctx.recv_cq) {
808 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
812 if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
813 fprintf(stderr, "Couldn't request CQ notification\n");
820 static void init(void)
822 srand48(getpid() * time(NULL));
828 ctx.mtu = IBV_MTU_2048;
833 static void usage(const char *argv0)
836 printf(" %s start a server and wait for connection\n", argv0);
837 printf(" %s <host> connect to server at <host>\n", argv0);
839 printf("Options:\n");
840 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
841 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
842 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
843 printf(" -s, --size=<size> size of message to exchange (default 4096)\n");
844 printf(" -m, --mtu=<size> path MTU (default 2048)\n");
845 printf(" -c, --clients=<n> number of clients (on server only, default 1)\n");
846 printf(" -n, --num_tests=<n> number of tests per client (default 5)\n");
847 printf(" -l, --sl=<sl> service level value\n");
848 printf(" -e, --events sleep on CQ events (default poll)\n");
849 printf(" -g, --gid-idx=<gid index> local port gid index\n");
852 int main(int argc, char *argv[])
854 char *ib_devname = NULL;
855 char *servername = NULL;
857 int i, total, cnt = 0;
858 int ne, qpi, num_cq_events = 0;
865 static struct option long_options[] = {
866 { .name = "port", .has_arg = 1, .val = 'p' },
867 { .name = "ib-dev", .has_arg = 1, .val = 'd' },
868 { .name = "ib-port", .has_arg = 1, .val = 'i' },
869 { .name = "size", .has_arg = 1, .val = 's' },
870 { .name = "mtu", .has_arg = 1, .val = 'm' },
871 { .name = "clients", .has_arg = 1, .val = 'c' },
872 { .name = "num_tests", .has_arg = 1, .val = 'n' },
873 { .name = "sl", .has_arg = 1, .val = 'l' },
874 { .name = "events", .has_arg = 0, .val = 'e' },
875 { .name = "gid-idx", .has_arg = 1, .val = 'g' },
879 c = getopt_long(argc, argv, "p:d:i:s:m:c:n:l:eg:", long_options,
886 port = strtol(optarg, NULL, 0);
887 if (port < 0 || port > 65535) {
893 ib_devname = strdupa(optarg);
896 ctx.ib_port = strtol(optarg, NULL, 0);
897 if (ctx.ib_port < 0) {
903 ctx.size = strtol(optarg, NULL, 0);
906 ctx.mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
913 ctx.num_clients = strtol(optarg, NULL, 0);
916 ctx.num_tests = strtol(optarg, NULL, 0);
919 ctx.sl = strtol(optarg, NULL, 0);
922 ctx.gidx = strtol(optarg, NULL, 0);
933 if (optind == argc - 1) {
934 servername = strdupa(argv[optind]);
936 } else if (optind < argc) {
941 page_size = sysconf(_SC_PAGESIZE);
943 if (pp_init_ctx(ib_devname))
946 if (pp_post_recv(ctx.num_clients)) {
947 fprintf(stderr, "Couldn't post receives\n");
952 if (pp_client_connect(servername, port))
955 if (pp_server_connect(port))
958 for (i = 0; i < ctx.num_clients; i++)
962 total = ctx.num_clients * ctx.num_tests;
963 while (cnt < total) {
972 ne = ibv_poll_cq(ctx.recv_cq, 1, &wc);
974 fprintf(stderr, "Error polling cq %d\n", ne);
976 } else if (ne == 0) {
981 fprintf(stderr, "Work completion error %d\n", wc.status);
986 qpi = find_qp(wc.qp_num);
987 if (ctx.rem_dest[qpi].pp_cnt < ctx.num_tests)
993 for (cnt = 0; cnt < ctx.num_clients; cnt += ne) {
994 ne = ibv_poll_cq(ctx.send_cq, 1, &wc);
996 fprintf(stderr, "Error polling cq %d\n", ne);
1002 ibv_ack_cq_events(ctx.recv_cq, num_cq_events);
1004 /* Process should get an ack from the daemon to close its resources to
1005 * make sure latest daemon's response sent via its target QP destined
1006 * to an XSRQ created by another client won't be lost.
1007 * Failure to do so may cause the other client to wait for that sent
1008 * message forever. See comment on pp_post_send.
1011 if (pp_client_termination())
1013 } else if (pp_server_termination()) {
1020 printf("success\n");