2 * Copyright (c) 2012 Intel Corporation. All rights reserved.
4 * This software is available to you under the OpenIB.org BSD license
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * - Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * - Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
23 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
24 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
25 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
26 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <sys/types.h>
37 #include <sys/socket.h>
43 #include <netinet/tcp.h>
44 #include <arpa/inet.h>
46 #include <rdma/rdma_cma.h>
47 #include <rdma/rsocket.h>
48 #include <util/compiler.h>
51 static int test_size[] = {
53 (1 << 7), ((1 << 7) + (1 << 6)),
54 (1 << 8), ((1 << 8) + (1 << 7)),
55 (1 << 9), ((1 << 9) + (1 << 8)),
56 (1 << 10), ((1 << 10) + (1 << 9)),
58 #define TEST_CNT (sizeof test_size / sizeof test_size[0])
77 #define CTRL_MSG_SIZE 16
83 static struct client clients[256];
88 static int flags = MSG_DONTWAIT;
89 static int poll_timeout;
92 static int transfer_size = 1000;
93 static int transfer_count = 1000;
94 static int buffer_size;
95 static char test_name[10] = "custom";
96 static const char *port = "7174";
97 static char *dst_addr;
98 static char *src_addr;
99 static union socket_addr g_addr;
100 static socklen_t g_addrlen;
101 static struct timeval start, end;
102 static struct message g_msg;
104 static void show_perf(void)
111 usec = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec);
112 transfers = echo ? transfer_count * 2 : be32toh(g_msg.data);
113 bytes = (long long) transfers * transfer_size;
115 /* name size transfers bytes seconds Gb/sec usec/xfer */
116 printf("%-10s", test_name);
117 size_str(str, sizeof str, transfer_size);
119 cnt_str(str, sizeof str, transfers);
121 size_str(str, sizeof str, bytes);
123 printf("%8.2fs%10.2f%11.2f\n",
124 usec / 1000000., (bytes * 8) / (1000. * usec),
128 static void init_latency_test(int size)
132 size_str(sstr, sizeof sstr, size);
133 snprintf(test_name, sizeof test_name, "%s_lat", sstr);
134 transfer_size = size;
135 transfer_count = size_to_count(transfer_size) / 10;
139 static void init_bandwidth_test(int size)
143 size_str(sstr, sizeof sstr, size);
144 snprintf(test_name, sizeof test_name, "%s_bw", sstr);
145 transfer_size = size;
146 transfer_count = size_to_count(transfer_size);
150 static void set_options(int fd)
155 rs_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (void *) &buffer_size,
157 rs_setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (void *) &buffer_size,
161 rs_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (void *) &val, sizeof val);
162 rs_setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (void *) &val, sizeof val);
165 if (flags & MSG_DONTWAIT)
166 rs_fcntl(fd, F_SETFL, O_NONBLOCK);
169 static ssize_t svr_send(struct message *msg, size_t size,
170 union socket_addr *addr, socklen_t addrlen)
177 fds.events = POLLOUT;
182 ret = do_poll(&fds, poll_timeout);
187 ret = rs_sendto(rs, msg, size, flags, &addr->sa, addrlen);
188 } while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
196 static ssize_t svr_recv(struct message *msg, size_t size,
197 union socket_addr *addr, socklen_t *addrlen)
209 ret = do_poll(&fds, poll_timeout);
214 ret = rs_recvfrom(rs, msg, size, flags, &addr->sa, addrlen);
215 } while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
223 static int svr_process(struct message *msg, size_t size,
224 union socket_addr *addr, socklen_t addrlen)
231 if (addr->sa.sa_family == AF_INET) {
232 printf("client login from %s\n",
233 inet_ntop(AF_INET, &addr->sin.sin_addr.s_addr,
236 printf("client login from %s\n",
237 inet_ntop(AF_INET6, &addr->sin6.sin6_addr.s6_addr,
243 memset(&clients[msg->id], 0, sizeof clients[msg->id]);
246 clients[msg->id].recvcnt++;
249 msg->data = htobe32(clients[msg->id].recvcnt);
252 clients[msg->id].recvcnt++;
256 ret = svr_send(msg, size, addr, addrlen);
257 return (ret == size) ? 0 : (int) ret;
260 static int svr_bind(void)
262 struct addrinfo hints, *res;
265 memset(&hints, 0, sizeof hints);
266 hints.ai_socktype = SOCK_DGRAM;
267 ret = getaddrinfo(src_addr, port, &hints, &res);
269 printf("getaddrinfo: %s\n", gai_strerror(ret));
273 rs = rs_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
281 ret = rs_bind(rs, res->ai_addr, res->ai_addrlen);
292 static int svr_run(void)
299 g_addrlen = sizeof g_addr;
300 len = svr_recv(&g_msg, sizeof g_msg, &g_addr, &g_addrlen);
304 ret = svr_process(&g_msg, len, &g_addr, g_addrlen);
309 static ssize_t client_send(struct message *msg, size_t size)
316 fds.events = POLLOUT;
321 ret = do_poll(&fds, poll_timeout);
326 ret = rs_send(rs, msg, size, flags);
327 } while (ret < 0 && (errno == EWOULDBLOCK || errno == EAGAIN));
335 static ssize_t client_recv(struct message *msg, size_t size, int timeout)
344 ret = rs_poll(&fds, 1, timeout);
349 ret = rs_recv(rs, msg, size, flags | MSG_DONTWAIT);
350 if (ret < 0 && errno != EWOULDBLOCK && errno != EAGAIN)
356 static int client_send_recv(struct message *msg, size_t size, int timeout)
358 static uint8_t seqno;
363 ret = client_send(msg, size);
367 ret = client_recv(msg, size, timeout);
368 } while (ret <= 0 || msg->seqno != seqno);
374 static int run_test(void)
378 g_msg.op = msg_op_start;
379 ret = client_send_recv(&g_msg, CTRL_MSG_SIZE, 1000);
380 if (ret != CTRL_MSG_SIZE)
383 g_msg.op = echo ? msg_op_echo : msg_op_data;
384 gettimeofday(&start, NULL);
385 for (i = 0; i < transfer_count; i++) {
386 ret = echo ? client_send_recv(&g_msg, transfer_size, 1) :
387 client_send(&g_msg, transfer_size);
388 if (ret != transfer_size)
392 g_msg.op = msg_op_end;
393 ret = client_send_recv(&g_msg, CTRL_MSG_SIZE, 1);
394 if (ret != CTRL_MSG_SIZE)
397 gettimeofday(&end, NULL);
405 static int client_connect(void)
407 struct addrinfo hints, *res;
410 memset(&hints, 0, sizeof hints);
411 hints.ai_socktype = SOCK_DGRAM;
412 ret = getaddrinfo(dst_addr, port, &hints, &res);
414 printf("getaddrinfo: %s\n", gai_strerror(ret));
418 rs = rs_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
426 ret = rs_connect(rs, res->ai_addr, res->ai_addrlen);
433 g_msg.op = msg_op_login;
434 ret = client_send_recv(&g_msg, CTRL_MSG_SIZE, 1000);
435 if (ret == CTRL_MSG_SIZE)
443 static int client_run(void)
447 printf("%-10s%-8s%-8s%-8s%8s %10s%13s\n",
448 "name", "bytes", "xfers", "total", "time", "Gb/sec", "usec/xfer");
450 ret = client_connect();
455 for (i = 0; i < TEST_CNT; i++) {
456 init_latency_test(test_size[i]);
459 for (i = 0; i < TEST_CNT; i++) {
460 init_bandwidth_test(test_size[i]);
471 static int set_test_opt(const char *arg)
473 if (strlen(arg) == 1) {
485 flags = MSG_DONTWAIT;
494 if (!strncasecmp("socket", arg, 6)) {
496 } else if (!strncasecmp("async", arg, 5)) {
498 } else if (!strncasecmp("block", arg, 5)) {
500 } else if (!strncasecmp("nonblock", arg, 8)) {
501 flags = MSG_DONTWAIT;
502 } else if (!strncasecmp("echo", arg, 4)) {
511 int main(int argc, char **argv)
515 while ((op = getopt(argc, argv, "s:b:B:C:S:p:T:")) != -1) {
524 buffer_size = atoi(optarg);
528 transfer_count = atoi(optarg);
532 transfer_size = atoi(optarg);
533 if (transfer_size < CTRL_MSG_SIZE) {
534 printf("size must be at least %d bytes\n",
543 if (!set_test_opt(optarg))
545 /* invalid option - fall through */
548 printf("usage: %s\n", argv[0]);
549 printf("\t[-s server_address]\n");
550 printf("\t[-b bind_address]\n");
551 printf("\t[-B buffer_size]\n");
552 printf("\t[-C transfer_count]\n");
553 printf("\t[-S transfer_size]\n");
554 printf("\t[-p port_number]\n");
555 printf("\t[-T test_option]\n");
556 printf("\t s|sockets - use standard tcp/ip sockets\n");
557 printf("\t a|async - asynchronous operation (use poll)\n");
558 printf("\t b|blocking - use blocking calls\n");
559 printf("\t n|nonblocking - use nonblocking calls\n");
560 printf("\t e|echo - server echoes all messages\n");
568 ret = dst_addr ? client_run() : svr_run();