2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <sys/types.h>
40 #include <sys/socket.h>
45 #include <arpa/inet.h>
51 #include <sys/param.h>
54 PINGPONG_RECV_WRID = 1,
55 PINGPONG_SEND_WRID = 2,
62 struct pingpong_context {
63 struct ibv_context *context;
64 struct ibv_comp_channel *channel;
69 struct ibv_cq_ex *cq_ex;
77 struct ibv_port_attr portinfo;
78 uint64_t completion_timestamp_mask;
81 static struct ibv_cq *pp_cq(struct pingpong_context *ctx)
83 return use_ts ? ibv_cq_ex_to_cq(ctx->cq_s.cq_ex) :
87 struct pingpong_dest {
94 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
95 enum ibv_mtu mtu, int sl,
96 struct pingpong_dest *dest, int sgid_idx)
98 struct ibv_qp_attr attr = {
99 .qp_state = IBV_QPS_RTR,
101 .dest_qp_num = dest->qpn,
103 .max_dest_rd_atomic = 1,
114 if (dest->gid.global.interface_id) {
115 attr.ah_attr.is_global = 1;
116 attr.ah_attr.grh.hop_limit = 1;
117 attr.ah_attr.grh.dgid = dest->gid;
118 attr.ah_attr.grh.sgid_index = sgid_idx;
120 if (ibv_modify_qp(ctx->qp, &attr,
126 IBV_QP_MAX_DEST_RD_ATOMIC |
127 IBV_QP_MIN_RNR_TIMER)) {
128 fprintf(stderr, "Failed to modify QP to RTR\n");
132 attr.qp_state = IBV_QPS_RTS;
136 attr.sq_psn = my_psn;
137 attr.max_rd_atomic = 1;
138 if (ibv_modify_qp(ctx->qp, &attr,
144 IBV_QP_MAX_QP_RD_ATOMIC)) {
145 fprintf(stderr, "Failed to modify QP to RTS\n");
152 static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
153 const struct pingpong_dest *my_dest)
155 struct addrinfo *res, *t;
156 struct addrinfo hints = {
157 .ai_family = AF_UNSPEC,
158 .ai_socktype = SOCK_STREAM
161 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
164 struct pingpong_dest *rem_dest = NULL;
167 if (asprintf(&service, "%d", port) < 0)
170 n = getaddrinfo(servername, service, &hints, &res);
173 fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
178 for (t = res; t; t = t->ai_next) {
179 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
181 if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
188 freeaddrinfo_null(res);
192 fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
196 gid_to_wire_gid(&my_dest->gid, gid);
197 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
199 if (write(sockfd, msg, sizeof msg) != sizeof msg) {
200 fprintf(stderr, "Couldn't send local address\n");
204 if (read(sockfd, msg, sizeof msg) != sizeof msg ||
205 write(sockfd, "done", sizeof "done") != sizeof "done") {
206 perror("client read/write");
207 fprintf(stderr, "Couldn't read/write remote address\n");
211 rem_dest = malloc(sizeof *rem_dest);
215 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
216 &rem_dest->psn, gid);
217 wire_gid_to_gid(gid, &rem_dest->gid);
224 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
225 int ib_port, enum ibv_mtu mtu,
227 const struct pingpong_dest *my_dest,
230 struct addrinfo *res, *t;
231 struct addrinfo hints = {
232 .ai_flags = AI_PASSIVE,
233 .ai_family = AF_INET,
234 .ai_socktype = SOCK_STREAM
237 char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
239 int sockfd = -1, connfd;
240 struct pingpong_dest *rem_dest = NULL;
243 if (asprintf(&service, "%d", port) < 0)
246 n = getaddrinfo(NULL, service, &hints, &res);
249 fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
254 for (t = res; t; t = t->ai_next) {
255 sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
259 setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
261 if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
268 freeaddrinfo_null(res);
272 fprintf(stderr, "Couldn't listen to port %d\n", port);
277 connfd = accept(sockfd, NULL, NULL);
280 fprintf(stderr, "accept() failed\n");
284 n = read(connfd, msg, sizeof msg);
285 if (n != sizeof msg) {
286 perror("server read");
287 fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
291 rem_dest = malloc(sizeof *rem_dest);
295 sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
296 &rem_dest->psn, gid);
297 wire_gid_to_gid(gid, &rem_dest->gid);
299 if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest,
301 fprintf(stderr, "Couldn't connect to remote QP\n");
308 gid_to_wire_gid(&my_dest->gid, gid);
309 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
311 if (write(connfd, msg, sizeof msg) != sizeof msg ||
312 read(connfd, msg, sizeof msg) != sizeof "done") {
313 fprintf(stderr, "Couldn't send/recv local address\n");
325 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
326 int rx_depth, int port,
329 struct pingpong_context *ctx;
330 int access_flags = IBV_ACCESS_LOCAL_WRITE;
332 ctx = calloc(1, sizeof *ctx);
337 ctx->send_flags = IBV_SEND_SIGNALED;
338 ctx->rx_depth = rx_depth;
340 ctx->buf = memalign(page_size, size);
342 fprintf(stderr, "Couldn't allocate work buf.\n");
346 /* FIXME memset(ctx->buf, 0, size); */
347 memset(ctx->buf, 0x7b, size);
349 ctx->context = ibv_open_device(ib_dev);
351 fprintf(stderr, "Couldn't get context for %s\n",
352 ibv_get_device_name(ib_dev));
357 ctx->channel = ibv_create_comp_channel(ctx->context);
359 fprintf(stderr, "Couldn't create completion channel\n");
365 ctx->pd = ibv_alloc_pd(ctx->context);
367 fprintf(stderr, "Couldn't allocate PD\n");
368 goto clean_comp_channel;
371 if (use_odp || use_ts) {
372 const uint32_t rc_caps_mask = IBV_ODP_SUPPORT_SEND |
373 IBV_ODP_SUPPORT_RECV;
374 struct ibv_device_attr_ex attrx;
376 if (ibv_query_device_ex(ctx->context, NULL, &attrx)) {
377 fprintf(stderr, "Couldn't query device for its features\n");
378 goto clean_comp_channel;
382 if (!(attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
383 (attrx.odp_caps.per_transport_caps.rc_odp_caps & rc_caps_mask) != rc_caps_mask) {
384 fprintf(stderr, "The device isn't ODP capable or does not support RC send and receive with ODP\n");
385 goto clean_comp_channel;
387 access_flags |= IBV_ACCESS_ON_DEMAND;
391 if (!attrx.completion_timestamp_mask) {
392 fprintf(stderr, "The device isn't completion timestamp capable\n");
393 goto clean_comp_channel;
395 ctx->completion_timestamp_mask = attrx.completion_timestamp_mask;
398 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, access_flags);
401 fprintf(stderr, "Couldn't register MR\n");
406 struct ibv_cq_init_attr_ex attr_ex = {
409 .channel = ctx->channel,
411 .wc_flags = IBV_WC_EX_WITH_COMPLETION_TIMESTAMP
414 ctx->cq_s.cq_ex = ibv_create_cq_ex(ctx->context, &attr_ex);
416 ctx->cq_s.cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
421 fprintf(stderr, "Couldn't create CQ\n");
426 struct ibv_qp_attr attr;
427 struct ibv_qp_init_attr init_attr = {
428 .send_cq = pp_cq(ctx),
429 .recv_cq = pp_cq(ctx),
432 .max_recv_wr = rx_depth,
436 .qp_type = IBV_QPT_RC
439 ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
441 fprintf(stderr, "Couldn't create QP\n");
445 ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
446 if (init_attr.cap.max_inline_data >= size) {
447 ctx->send_flags |= IBV_SEND_INLINE;
452 struct ibv_qp_attr attr = {
453 .qp_state = IBV_QPS_INIT,
459 if (ibv_modify_qp(ctx->qp, &attr,
463 IBV_QP_ACCESS_FLAGS)) {
464 fprintf(stderr, "Failed to modify QP to INIT\n");
472 ibv_destroy_qp(ctx->qp);
475 ibv_destroy_cq(pp_cq(ctx));
478 ibv_dereg_mr(ctx->mr);
481 ibv_dealloc_pd(ctx->pd);
485 ibv_destroy_comp_channel(ctx->channel);
488 ibv_close_device(ctx->context);
499 static int pp_close_ctx(struct pingpong_context *ctx)
501 if (ibv_destroy_qp(ctx->qp)) {
502 fprintf(stderr, "Couldn't destroy QP\n");
506 if (ibv_destroy_cq(pp_cq(ctx))) {
507 fprintf(stderr, "Couldn't destroy CQ\n");
511 if (ibv_dereg_mr(ctx->mr)) {
512 fprintf(stderr, "Couldn't deregister MR\n");
516 if (ibv_dealloc_pd(ctx->pd)) {
517 fprintf(stderr, "Couldn't deallocate PD\n");
522 if (ibv_destroy_comp_channel(ctx->channel)) {
523 fprintf(stderr, "Couldn't destroy completion channel\n");
528 if (ibv_close_device(ctx->context)) {
529 fprintf(stderr, "Couldn't release context\n");
539 static int pp_post_recv(struct pingpong_context *ctx, int n)
541 struct ibv_sge list = {
542 .addr = (uintptr_t) ctx->buf,
544 .lkey = ctx->mr->lkey
546 struct ibv_recv_wr wr = {
547 .wr_id = PINGPONG_RECV_WRID,
551 struct ibv_recv_wr *bad_wr;
554 for (i = 0; i < n; ++i)
555 if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
561 static int pp_post_send(struct pingpong_context *ctx)
563 struct ibv_sge list = {
564 .addr = (uintptr_t) ctx->buf,
566 .lkey = ctx->mr->lkey
568 struct ibv_send_wr wr = {
569 .wr_id = PINGPONG_SEND_WRID,
572 .opcode = IBV_WR_SEND,
573 .send_flags = ctx->send_flags,
575 struct ibv_send_wr *bad_wr;
577 return ibv_post_send(ctx->qp, &wr, &bad_wr);
581 uint64_t comp_recv_max_time_delta;
582 uint64_t comp_recv_min_time_delta;
583 uint64_t comp_recv_total_time_delta;
584 uint64_t comp_recv_prev_time;
585 int last_comp_with_ts;
586 unsigned int comp_with_time_iters;
589 static inline int parse_single_wc(struct pingpong_context *ctx, int *scnt,
590 int *rcnt, int *routs, int iters,
591 uint64_t wr_id, enum ibv_wc_status status,
592 uint64_t completion_timestamp,
593 struct ts_params *ts)
595 if (status != IBV_WC_SUCCESS) {
596 fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
597 ibv_wc_status_str(status),
602 switch ((int)wr_id) {
603 case PINGPONG_SEND_WRID:
607 case PINGPONG_RECV_WRID:
608 if (--(*routs) <= 1) {
609 *routs += pp_post_recv(ctx, ctx->rx_depth - *routs);
610 if (*routs < ctx->rx_depth) {
612 "Couldn't post receive (%d)\n",
620 if (ts->last_comp_with_ts) {
623 /* checking whether the clock was wrapped around */
624 if (completion_timestamp >= ts->comp_recv_prev_time)
625 delta = completion_timestamp - ts->comp_recv_prev_time;
627 delta = ctx->completion_timestamp_mask - ts->comp_recv_prev_time +
628 completion_timestamp + 1;
630 ts->comp_recv_max_time_delta = MAX(ts->comp_recv_max_time_delta, delta);
631 ts->comp_recv_min_time_delta = MIN(ts->comp_recv_min_time_delta, delta);
632 ts->comp_recv_total_time_delta += delta;
633 ts->comp_with_time_iters++;
636 ts->comp_recv_prev_time = completion_timestamp;
637 ts->last_comp_with_ts = 1;
639 ts->last_comp_with_ts = 0;
645 fprintf(stderr, "Completion for unknown wr_id %d\n",
650 ctx->pending &= ~(int)wr_id;
651 if (*scnt < iters && !ctx->pending) {
652 if (pp_post_send(ctx)) {
653 fprintf(stderr, "Couldn't post send\n");
656 ctx->pending = PINGPONG_RECV_WRID |
663 static void usage(const char *argv0)
666 printf(" %s start a server and wait for connection\n", argv0);
667 printf(" %s <host> connect to server at <host>\n", argv0);
669 printf("Options:\n");
670 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
671 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
672 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
673 printf(" -s, --size=<size> size of message to exchange (default 4096)\n");
674 printf(" -m, --mtu=<size> path MTU (default 1024)\n");
675 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
676 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
677 printf(" -l, --sl=<sl> service level value\n");
678 printf(" -e, --events sleep on CQ events (default poll)\n");
679 printf(" -g, --gid-idx=<gid index> local port gid index\n");
680 printf(" -o, --odp use on demand paging\n");
681 printf(" -t, --ts get CQE with timestamp\n");
684 int main(int argc, char *argv[])
686 struct ibv_device **dev_list;
687 struct ibv_device *ib_dev;
688 struct pingpong_context *ctx;
689 struct pingpong_dest my_dest;
690 struct pingpong_dest *rem_dest;
691 struct timeval start, end;
692 char *ib_devname = NULL;
693 char *servername = NULL;
694 unsigned int port = 18515;
696 unsigned int size = 4096;
697 enum ibv_mtu mtu = IBV_MTU_1024;
698 unsigned int rx_depth = 500;
699 unsigned int iters = 1000;
703 int num_cq_events = 0;
709 srand48(getpid() * time(NULL));
714 static struct option long_options[] = {
715 { .name = "port", .has_arg = 1, .val = 'p' },
716 { .name = "ib-dev", .has_arg = 1, .val = 'd' },
717 { .name = "ib-port", .has_arg = 1, .val = 'i' },
718 { .name = "size", .has_arg = 1, .val = 's' },
719 { .name = "mtu", .has_arg = 1, .val = 'm' },
720 { .name = "rx-depth", .has_arg = 1, .val = 'r' },
721 { .name = "iters", .has_arg = 1, .val = 'n' },
722 { .name = "sl", .has_arg = 1, .val = 'l' },
723 { .name = "events", .has_arg = 0, .val = 'e' },
724 { .name = "gid-idx", .has_arg = 1, .val = 'g' },
725 { .name = "odp", .has_arg = 0, .val = 'o' },
726 { .name = "ts", .has_arg = 0, .val = 't' },
730 c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:ot",
738 port = strtoul(optarg, NULL, 0);
746 ib_devname = strdupa(optarg);
750 ib_port = strtol(optarg, NULL, 0);
758 size = strtoul(optarg, NULL, 0);
762 mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
770 rx_depth = strtoul(optarg, NULL, 0);
774 iters = strtoul(optarg, NULL, 0);
778 sl = strtol(optarg, NULL, 0);
786 gidx = strtol(optarg, NULL, 0);
802 if (optind == argc - 1)
803 servername = strdupa(argv[optind]);
804 else if (optind < argc) {
810 ts.comp_recv_max_time_delta = 0;
811 ts.comp_recv_min_time_delta = 0xffffffff;
812 ts.comp_recv_total_time_delta = 0;
813 ts.comp_recv_prev_time = 0;
814 ts.last_comp_with_ts = 0;
815 ts.comp_with_time_iters = 0;
818 page_size = sysconf(_SC_PAGESIZE);
820 dev_list = ibv_get_device_list(NULL);
822 perror("Failed to get IB devices list");
829 fprintf(stderr, "No IB devices found\n");
834 for (i = 0; dev_list[i]; ++i)
835 if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
837 ib_dev = dev_list[i];
839 fprintf(stderr, "IB device %s not found\n", ib_devname);
844 ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
848 routs = pp_post_recv(ctx, ctx->rx_depth);
849 if (routs < ctx->rx_depth) {
850 fprintf(stderr, "Couldn't post receive (%d)\n", routs);
855 if (ibv_req_notify_cq(pp_cq(ctx), 0)) {
856 fprintf(stderr, "Couldn't request CQ notification\n");
861 if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
862 fprintf(stderr, "Couldn't get port info\n");
866 my_dest.lid = ctx->portinfo.lid;
867 if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET &&
869 fprintf(stderr, "Couldn't get local LID\n");
874 if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
875 fprintf(stderr, "can't read sgid of index %d\n", gidx);
879 memset(&my_dest.gid, 0, sizeof my_dest.gid);
881 my_dest.qpn = ctx->qp->qp_num;
882 my_dest.psn = lrand48() & 0xffffff;
883 inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
884 printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
885 my_dest.lid, my_dest.qpn, my_dest.psn, gid);
889 rem_dest = pp_client_exch_dest(servername, port, &my_dest);
891 rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl,
897 inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
898 printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
899 rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
902 if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest,
906 ctx->pending = PINGPONG_RECV_WRID;
909 if (pp_post_send(ctx)) {
910 fprintf(stderr, "Couldn't post send\n");
913 ctx->pending |= PINGPONG_SEND_WRID;
916 if (gettimeofday(&start, NULL)) {
917 perror("gettimeofday");
922 while (rcnt < iters || scnt < iters) {
926 struct ibv_cq *ev_cq;
929 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
930 fprintf(stderr, "Failed to get cq_event\n");
936 if (ev_cq != pp_cq(ctx)) {
937 fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
941 if (ibv_req_notify_cq(pp_cq(ctx), 0)) {
942 fprintf(stderr, "Couldn't request CQ notification\n");
948 struct ibv_poll_cq_attr attr = {};
951 ret = ibv_start_poll(ctx->cq_s.cq_ex, &attr);
952 } while (!use_event && ret == ENOENT);
955 fprintf(stderr, "poll CQ failed %d\n", ret);
958 ret = parse_single_wc(ctx, &scnt, &rcnt, &routs,
960 ctx->cq_s.cq_ex->wr_id,
961 ctx->cq_s.cq_ex->status,
962 ibv_wc_read_completion_ts(ctx->cq_s.cq_ex),
965 ibv_end_poll(ctx->cq_s.cq_ex);
968 ret = ibv_next_poll(ctx->cq_s.cq_ex);
970 ret = parse_single_wc(ctx, &scnt, &rcnt, &routs,
972 ctx->cq_s.cq_ex->wr_id,
973 ctx->cq_s.cq_ex->status,
974 ibv_wc_read_completion_ts(ctx->cq_s.cq_ex),
976 ibv_end_poll(ctx->cq_s.cq_ex);
977 if (ret && ret != ENOENT) {
978 fprintf(stderr, "poll CQ failed %d\n", ret);
986 ne = ibv_poll_cq(pp_cq(ctx), 2, wc);
988 fprintf(stderr, "poll CQ failed %d\n", ne);
991 } while (!use_event && ne < 1);
993 for (i = 0; i < ne; ++i) {
994 ret = parse_single_wc(ctx, &scnt, &rcnt, &routs,
1000 fprintf(stderr, "parse WC failed %d\n", ne);
1007 if (gettimeofday(&end, NULL)) {
1008 perror("gettimeofday");
1013 float usec = (end.tv_sec - start.tv_sec) * 1000000 +
1014 (end.tv_usec - start.tv_usec);
1015 long long bytes = (long long) size * iters * 2;
1017 printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
1018 bytes, usec / 1000000., bytes * 8. / usec);
1019 printf("%d iters in %.2f seconds = %.2f usec/iter\n",
1020 iters, usec / 1000000., usec / iters);
1022 if (use_ts && ts.comp_with_time_iters) {
1023 printf("Max receive completion clock cycles = %" PRIu64 "\n",
1024 ts.comp_recv_max_time_delta);
1025 printf("Min receive completion clock cycles = %" PRIu64 "\n",
1026 ts.comp_recv_min_time_delta);
1027 printf("Average receive completion clock cycles = %f\n",
1028 (double)ts.comp_recv_total_time_delta / ts.comp_with_time_iters);
1032 ibv_ack_cq_events(pp_cq(ctx), num_cq_events);
1034 if (pp_close_ctx(ctx))
1037 ibv_free_device_list(dev_list);