2 * Copyright (c) 2005 Robert N. M. Watson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/types.h>
28 #include <sys/socket.h>
29 #include <sys/stdint.h>
31 #include <sys/utsname.h>
34 #include <netinet/in.h>
46 * juggle is a simple IPC/context switch performance test, which works on
47 * pairs of file descriptors of various types. In various runs, it considers
48 * the cost of bouncing a message synchronously across the descriptor pair,
49 * either in the same thread, two different threads, or two different
50 * processes. Timing measurements for each series of I/O's are reported, but
51 * the first measurement in each series discarded as "warmup" on the IPC
52 * primitive. Variations on the test permit for pipelining, or the insertion
53 * of more than one packet into the stream at a time, intended to permit
54 * greater parallelism, hopefully allowing performance numbers to reflect
55 * use of available parallelism, and/or intelligence in context switching to
56 * avoid premature switching when multiple messages are queued.
60 * The UDP test uses UDP over the loopback interface. Two arbitrary but
63 #define UDP_PORT1 2020
64 #define UDP_PORT2 2021
67 * Size of each message. Must be smaller than the socket buffer or pipe
68 * buffer maximum size, as we want to send it atomically without blocking.
69 * If pipelining is in use, must be able to fit PIPELINE_MAX of these
70 * messages into the send queue.
72 #define MESSAGELEN 128
75 * Number of message cycles -- into fd1, out of fd2, into fd2, and out of
76 * fd1. By counting in cycles, we allow the master thread or process to
77 * perform timing without explicitly synchronizing with the secondary thread
80 #define NUMCYCLES 1024
83 * Number of times to run each test.
88 * Number of in-flight messages per cycle. I adjusting this value, be
89 * careful not to exceed the socket/etc buffer depth, or messages may be lost
90 * or result in blocking.
92 #define PIPELINE_MAX 4
95 udp_create(int *fd1p, int *fd2p)
97 struct sockaddr_in sin1, sin2;
100 sock1 = socket(PF_INET, SOCK_DGRAM, 0);
104 sock2 = socket(PF_INET, SOCK_DGRAM, 0);
110 bzero(&sin1, sizeof(sin1));
111 sin1.sin_len = sizeof(sin1);
112 sin1.sin_family = AF_INET;
113 sin1.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
114 sin1.sin_port = htons(UDP_PORT1);
116 bzero(&sin2, sizeof(sin2));
117 sin2.sin_len = sizeof(sin2);
118 sin2.sin_family = AF_INET;
119 sin2.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
120 sin2.sin_port = htons(UDP_PORT2);
122 if (bind(sock1, (struct sockaddr *) &sin1, sizeof(sin1)) < 0) {
128 if (bind(sock2, (struct sockaddr *) &sin2, sizeof(sin2)) < 0) {
134 if (connect(sock1, (struct sockaddr *) &sin2, sizeof(sin2)) < 0) {
140 if (connect(sock2, (struct sockaddr *) &sin1, sizeof(sin1)) < 0) {
153 pipe_create(int *fd1p, int *fd2p)
167 socketpairdgram_create(int *fd1p, int *fd2p)
171 if (socketpair(PF_LOCAL, SOCK_DGRAM, 0, fds) < 0)
181 socketpairstream_create(int *fd1p, int *fd2p)
185 if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fds) < 0)
197 u_char buffer[MESSAGELEN];
200 bzero(buffer, sizeof(buffer));
202 len = write(s, buffer, sizeof(buffer));
205 if (len != sizeof(buffer)) {
215 u_char buffer[MESSAGELEN];
218 len = read(s, buffer, sizeof(buffer));
221 if (len != sizeof(buffer)) {
229 * Juggle messages between two file descriptors in a single thread/process,
230 * so simply a measure of IPC performance.
232 static struct timespec
233 juggle(int fd1, int fd2, int pipeline)
235 struct timespec tstart, tfinish;
238 if (clock_gettime(CLOCK_REALTIME, &tstart) < 0)
239 err(-1, "juggle: clock_gettime");
241 for (i = 0; i < NUMCYCLES; i++) {
243 for (j = 0; j < pipeline; j++) {
244 if (message_send(fd1) < 0)
245 err(-1, "message_send fd1");
248 for (j = 0; j < pipeline; j++) {
249 if (message_recv(fd2) < 0)
250 err(-1, "message_recv fd2");
252 if (message_send(fd2) < 0)
253 err(-1, "message_send fd2");
256 for (j = 0; j < pipeline; j++) {
257 if (message_recv(fd1) < 0)
258 err(-1, "message_recv fd1");
262 if (clock_gettime(CLOCK_REALTIME, &tfinish) < 0)
263 err(-1, "juggle: clock_gettime");
265 timespecsub(&tfinish, &tstart, &tfinish);
271 * Juggle messages between two file descriptors in two threads, so measure
272 * the cost of IPC and the cost of a thread context switch.
274 * In order to avoid measuring thread creation time, we make use of a
275 * condition variable to decide when both threads are ready to begin
278 static int threaded_child_ready;
279 static pthread_mutex_t threaded_mtx;
280 static pthread_cond_t threaded_cond;
281 static int threaded_pipeline;
284 juggling_thread(void *arg)
290 if (pthread_mutex_lock(&threaded_mtx) != 0)
291 err(-1, "juggling_thread: pthread_mutex_lock");
293 threaded_child_ready = 1;
295 if (pthread_cond_signal(&threaded_cond) != 0)
296 err(-1, "juggling_thread: pthread_cond_signal");
298 if (pthread_mutex_unlock(&threaded_mtx) != 0)
299 err(-1, "juggling_thread: pthread_mutex_unlock");
301 for (i = 0; i < NUMCYCLES; i++) {
302 for (j = 0; j < threaded_pipeline; j++) {
303 if (message_recv(fd2) < 0)
304 err(-1, "message_recv fd2");
306 if (message_send(fd2) < 0)
307 err(-1, "message_send fd2");
314 static struct timespec
315 thread_juggle(int fd1, int fd2, int pipeline)
317 struct timespec tstart, tfinish;
321 threaded_pipeline = pipeline;
323 if (pthread_mutex_init(&threaded_mtx, NULL) != 0)
324 err(-1, "thread_juggle: pthread_mutex_init");
326 if (pthread_create(&thread, NULL, juggling_thread, &fd2) != 0)
327 err(-1, "thread_juggle: pthread_create");
329 if (pthread_mutex_lock(&threaded_mtx) != 0)
330 err(-1, "thread_juggle: pthread_mutex_lock");
332 while (!threaded_child_ready) {
333 if (pthread_cond_wait(&threaded_cond, &threaded_mtx) != 0)
334 err(-1, "thread_juggle: pthread_cond_wait");
337 if (pthread_mutex_unlock(&threaded_mtx) != 0)
338 err(-1, "thread_juggle: pthread_mutex_unlock");
340 if (clock_gettime(CLOCK_REALTIME, &tstart) < 0)
341 err(-1, "thread_juggle: clock_gettime");
343 for (i = 0; i < NUMCYCLES; i++) {
344 for (j = 0; j < pipeline; j++) {
345 if (message_send(fd1) < 0)
346 err(-1, "message_send fd1");
349 for (j = 0; j < pipeline; j++) {
350 if (message_recv(fd1) < 0)
351 err(-1, "message_recv fd1");
355 if (clock_gettime(CLOCK_REALTIME, &tfinish) < 0)
356 err(-1, "thread_juggle: clock_gettime");
358 if (pthread_join(thread, NULL) != 0)
359 err(-1, "thread_juggle: pthread_join");
361 timespecsub(&tfinish, &tstart, &tfinish);
367 * Juggle messages between two file descriptors in two processes, so measure
368 * the cost of IPC and the cost of a process context switch.
370 * Since we can't use a mutex between the processes, we simply do an extra
371 * write on the child to let the parent know that it's ready to start.
373 static struct timespec
374 process_juggle(int fd1, int fd2, int pipeline)
376 struct timespec tstart, tfinish;
377 pid_t pid, ppid, wpid;
384 err(-1, "process_juggle: fork");
387 if (message_send(fd2) < 0) {
391 err(-1, "process_juggle: child: message_send");
394 for (i = 0; i < NUMCYCLES; i++) {
395 for (j = 0; j < pipeline; j++) {
396 if (message_send(fd2) < 0)
397 err(-1, "message_send fd2");
399 if (message_recv(fd2) < 0)
400 err(-1, "message_recv fd2");
406 if (message_recv(fd1) < 0) {
410 err(-1, "process_juggle: parent: message_recv");
413 if (clock_gettime(CLOCK_REALTIME, &tstart) < 0)
414 err(-1, "process_juggle: clock_gettime");
416 for (i = 0; i < NUMCYCLES; i++) {
417 for (j = 0; j < pipeline; j++) {
418 if (message_send(fd1) < 0) {
422 err(-1, "message_send fd1");
426 for (j = 0; j < pipeline; j++) {
427 if (message_recv(fd1) < 0) {
431 err(-1, "message_recv fd1");
436 if (clock_gettime(CLOCK_REALTIME, &tfinish) < 0)
437 err(-1, "process_juggle: clock_gettime");
440 wpid = waitpid(pid, NULL, 0);
442 err(-1, "process_juggle: waitpid");
444 errx(-1, "process_juggle: waitpid: pid != wpid");
446 timespecsub(&tfinish, &tstart, &tfinish);
452 * When we print out results for larger pipeline sizes, we scale back by the
453 * depth of the pipeline. This generally means dividing by the pipeline
454 * depth. Except when it means dividing by zero.
457 scale_timespec(struct timespec *ts, int p)
467 static const struct ipctype {
468 int (*it_create)(int *fd1p, int *fd2p);
471 { pipe_create, "pipe" },
472 { udp_create, "udp" },
473 { socketpairdgram_create, "socketpairdgram" },
474 { socketpairstream_create, "socketpairstream" },
476 static const int ipctypes_len = (sizeof(ipctypes) / sizeof(struct ipctype));
479 main(int argc, char *argv[])
481 struct timespec juggle_results[LOOPS], process_results[LOOPS];
482 struct timespec thread_results[LOOPS];
483 int fd1, fd2, i, j, p;
486 printf("version, juggle.c %s\n", "$FreeBSD$");
490 printf("sysname, %s\n", uts.sysname);
491 printf("nodename, %s\n", uts.nodename);
492 printf("release, %s\n", uts.release);
493 printf("version, %s\n", uts.version);
494 printf("machine, %s\n", uts.machine);
497 printf("MESSAGELEN, %d\n", MESSAGELEN);
498 printf("NUMCYCLES, %d\n", NUMCYCLES);
499 printf("LOOPS, %d\n", LOOPS);
500 printf("PIPELINE_MAX, %d\n", PIPELINE_MAX);
503 printf("ipctype, test, pipeline_depth");
504 for (j = 0; j < LOOPS; j++)
505 printf(", data%d", j);
508 for (p = 0; p < PIPELINE_MAX + 1; p++) {
509 for (i = 0; i < ipctypes_len; i++) {
510 if (ipctypes[i].it_create(&fd1, &fd2) < 0)
511 err(-1, "main: %s", ipctypes[i].it_name);
514 * For each test, do one uncounted warmup, then LOOPS
515 * runs of the actual test.
518 for (j = 0; j < LOOPS; j++)
519 juggle_results[j] = juggle(fd1, fd2, p);
520 process_juggle(fd1, fd2, p);
521 for (j = 0; j < LOOPS; j++)
522 process_results[j] = process_juggle(fd1, fd2,
524 thread_juggle(fd1, fd2, p);
525 for (j = 0; j < LOOPS; j++)
526 thread_results[j] = thread_juggle(fd1, fd2,
528 for (j = 0; j < LOOPS; j++) {
529 thread_results[j].tv_sec = 0;
530 thread_results[j].tv_nsec = 0;
536 * When printing results for the round, normalize the results
537 * with respect to the pipeline depth. We're doing p times
538 * as much work, and are we taking p times as long?
540 for (i = 0; i < ipctypes_len; i++) {
541 printf("%s, juggle, %d, ", ipctypes[i].it_name, p);
542 for (j = 0; j < LOOPS; j++) {
545 scale_timespec(&juggle_results[j], p);
547 (intmax_t)juggle_results[j].tv_sec,
548 juggle_results[j].tv_nsec);
551 printf("%s, process_juggle, %d, ",
552 ipctypes[i].it_name, p);
553 for (j = 0; j < LOOPS; j++) {
556 scale_timespec(&process_results[j], p);
558 (intmax_t)process_results[j].tv_sec,
559 process_results[j].tv_nsec);
562 printf("%s, thread_juggle, %d, ",
563 ipctypes[i].it_name, p);
564 for (j = 0; j < LOOPS; j++) {
567 scale_timespec(&thread_results[j], p);
569 (intmax_t)thread_results[j].tv_sec,
570 thread_results[j].tv_nsec);