2 * Copyright (c) 2003-2006, Maxime Henrion <mux@FreeBSD.org>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/types.h>
47 * Simple stream API to make my life easier. If the fgetln() and
48 * funopen() functions were standard and if funopen() wasn't using
49 * wrong types for the function pointers, I could have just used
50 * stdio, but life sucks.
52 * For now, streams are always block-buffered.
56 * Try to quiet warnings as much as possible with GCC while staying
57 * compatible with other compilers.
60 #if defined(__GNUC__) && (__GNUC__ > 2 || __GNUC__ == 2 && __GNUC_MINOR__ >= 7)
61 #define __unused __attribute__((__unused__))
68 * Flags passed to the flush methods.
70 * STREAM_FLUSH_CLOSING is passed during the last flush call before
71 * closing a stream. This allows the zlib filter to emit the EOF
72 * marker as appropriate. In all other cases, STREAM_FLUSH_NORMAL
75 * These flags are completely unused in the default flush method,
76 * but they are very important for the flush method of the zlib
85 * This is because buf_new() will always allocate size + 1 bytes,
86 * so our buffer sizes will still be power of 2 values.
88 #define STREAM_BUFSIZ 1023
102 stream_readfn_t *readfn;
103 stream_writefn_t *writefn;
104 stream_closefn_t *closefn;
106 struct stream_filter *filter;
110 typedef int stream_filter_initfn_t(struct stream *, void *);
111 typedef void stream_filter_finifn_t(struct stream *);
112 typedef int stream_filter_flushfn_t(struct stream *, struct buf *,
114 typedef ssize_t stream_filter_fillfn_t(struct stream *, struct buf *);
116 struct stream_filter {
118 stream_filter_initfn_t *initfn;
119 stream_filter_finifn_t *finifn;
120 stream_filter_fillfn_t *fillfn;
121 stream_filter_flushfn_t *flushfn;
124 /* Low-level buffer API. */
125 #define buf_avail(buf) ((buf)->size - (buf)->off - (buf)->in)
126 #define buf_count(buf) ((buf)->in)
127 #define buf_size(buf) ((buf)->size)
129 static struct buf *buf_new(size_t);
130 static void buf_more(struct buf *, size_t);
131 static void buf_less(struct buf *, size_t);
132 static void buf_free(struct buf *);
133 static void buf_grow(struct buf *, size_t);
135 /* Internal stream functions. */
136 static ssize_t stream_fill(struct stream *);
137 static ssize_t stream_fill_default(struct stream *, struct buf *);
138 static int stream_flush_int(struct stream *, stream_flush_t);
139 static int stream_flush_default(struct stream *, struct buf *,
142 /* Filters specific functions. */
143 static struct stream_filter *stream_filter_lookup(stream_filter_t);
144 static int stream_filter_init(struct stream *, void *);
145 static void stream_filter_fini(struct stream *);
147 /* The zlib stream filter declarations. */
148 #define ZFILTER_EOF 1 /* Got Z_STREAM_END. */
158 static int zfilter_init(struct stream *, void *);
159 static void zfilter_fini(struct stream *);
160 static ssize_t zfilter_fill(struct stream *, struct buf *);
161 static int zfilter_flush(struct stream *, struct buf *,
164 /* The MD5 stream filter. */
170 static int md5filter_init(struct stream *, void *);
171 static void md5filter_fini(struct stream *);
172 static ssize_t md5filter_fill(struct stream *, struct buf *);
173 static int md5filter_flush(struct stream *, struct buf *,
176 /* The available stream filters. */
177 struct stream_filter stream_filters[] = {
202 /* Create a new buffer. */
208 buf = xmalloc(sizeof(struct buf));
210 * We keep one spare byte so that stream_getln() can put a '\0'
211 * there in case the stream doesn't have an ending newline.
213 buf->buf = xmalloc(size + 1);
221 * Grow the size of the buffer. If "need" is 0, bump its size to the
222 * next power of 2 value. Otherwise, bump it to the next power of 2
223 * value bigger than "need".
226 buf_grow(struct buf *buf, size_t need)
230 buf->size = buf->size * 2 + 1; /* Account for the spare byte. */
232 assert(need > buf->size);
233 while (buf->size < need)
234 buf->size = buf->size * 2 + 1;
236 buf->buf = xrealloc(buf->buf, buf->size + 1);
239 /* Make more room in the buffer if needed. */
241 buf_prewrite(struct buf *buf)
244 if (buf_count(buf) == buf_size(buf))
246 if (buf_count(buf) > 0 && buf_avail(buf) == 0) {
247 memmove(buf->buf, buf->buf + buf->off, buf_count(buf));
252 /* Account for "n" bytes being added in the buffer. */
254 buf_more(struct buf *buf, size_t n)
257 assert(n <= buf_avail(buf));
261 /* Account for "n" bytes having been read in the buffer. */
263 buf_less(struct buf *buf, size_t n)
266 assert(n <= buf_count(buf));
276 buf_free(struct buf *buf)
283 static struct stream *
284 stream_new(stream_readfn_t *readfn, stream_writefn_t *writefn,
285 stream_closefn_t *closefn)
287 struct stream *stream;
289 stream = xmalloc(sizeof(struct stream));
290 if (readfn == NULL && writefn == NULL) {
295 stream->rdbuf = buf_new(STREAM_BUFSIZ);
297 stream->rdbuf = NULL;
299 stream->wrbuf = buf_new(STREAM_BUFSIZ);
301 stream->wrbuf = NULL;
302 stream->cookie = NULL;
304 stream->readfn = readfn;
305 stream->writefn = writefn;
306 stream->closefn = closefn;
307 stream->filter = stream_filter_lookup(STREAM_FILTER_NULL);
308 stream->fdata = NULL;
313 /* Create a new stream associated with a void *. */
315 stream_open(void *cookie, stream_readfn_t *readfn, stream_writefn_t *writefn,
316 stream_closefn_t *closefn)
318 struct stream *stream;
320 stream = stream_new(readfn, writefn, closefn);
321 stream->cookie = cookie;
325 /* Associate a file descriptor with a stream. */
327 stream_open_fd(int fd, stream_readfn_t *readfn, stream_writefn_t *writefn,
328 stream_closefn_t *closefn)
330 struct stream *stream;
332 stream = stream_new(readfn, writefn, closefn);
333 stream->cookie = &stream->fd;
338 /* Like open() but returns a stream. */
340 stream_open_file(const char *path, int flags, ...)
342 struct stream *stream;
343 stream_readfn_t *readfn;
344 stream_writefn_t *writefn;
350 if (flags & O_CREAT) {
352 * GCC says I should not be using mode_t here since it's
353 * promoted to an int when passed through `...'.
355 mode = va_arg(ap, int);
356 fd = open(path, flags, mode);
358 fd = open(path, flags);
364 if (flags == O_RDONLY) {
365 readfn = stream_read_fd;
367 } else if (flags == O_WRONLY) {
369 writefn = stream_write_fd;
370 } else if (flags == O_RDWR) {
371 assert(flags == O_RDWR);
372 readfn = stream_read_fd;
373 writefn = stream_write_fd;
380 stream = stream_open_fd(fd, readfn, writefn, stream_close_fd);
386 /* Return the file descriptor associated with this stream, or -1. */
388 stream_fileno(struct stream *stream)
394 /* Convenience read function for file descriptors. */
396 stream_read_fd(void *cookie, void *buf, size_t size)
402 nbytes = read(fd, buf, size);
406 /* Convenience write function for file descriptors. */
408 stream_write_fd(void *cookie, const void *buf, size_t size)
414 nbytes = write(fd, buf, size);
418 /* Convenience close function for file descriptors. */
420 stream_close_fd(void *cookie)
429 /* Read some bytes from the stream. */
431 stream_read(struct stream *stream, void *buf, size_t size)
437 rdbuf = stream->rdbuf;
438 if (buf_count(rdbuf) == 0) {
439 ret = stream_fill(stream);
443 n = min(size, buf_count(rdbuf));
444 memcpy(buf, rdbuf->buf + rdbuf->off, n);
450 * Read a line from the stream and return a pointer to it.
452 * If "len" is non-NULL, the length of the string will be put into it.
453 * The pointer is only valid until the next stream API call. The line
454 * can be modified by the caller, provided he doesn't write before or
457 * This is somewhat similar to the BSD fgetln() function, except that
458 * "len" can be NULL here. In that case the string is terminated by
459 * overwriting the '\n' character with a NUL character. If it's the
460 * last line in the stream and it has no ending newline, we can still
461 * add '\0' after it, because we keep one spare byte in the buffers.
463 * However, be warned that one can't handle binary lines properly
464 * without knowing the size of the string since those can contain
468 stream_getln(struct stream *stream, size_t *len)
476 if (buf_count(buf) == 0) {
477 n = stream_fill(stream);
481 cp = memchr(buf->buf + buf->off, '\n', buf_count(buf));
482 for (done = buf_count(buf); cp == NULL; done += n) {
483 n = stream_fill(stream);
487 /* Last line of the stream. */
488 cp = buf->buf + buf->off + buf->in - 1;
490 cp = memchr(buf->buf + buf->off + done, '\n',
491 buf_count(buf) - done);
493 line = buf->buf + buf->off;
495 size = cp - line + 1;
500 /* Terminate the string when len == NULL. */
501 if (line[size - 1] == '\n')
502 line[size - 1] = '\0';
509 /* Write some bytes to a stream. */
511 stream_write(struct stream *stream, const void *src, size_t nbytes)
517 if (nbytes > buf_size(buf))
518 buf_grow(buf, nbytes);
519 if (nbytes > buf_avail(buf)) {
520 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
524 memcpy(buf->buf + buf->off + buf->in, src, nbytes);
525 buf_more(buf, nbytes);
529 /* Formatted output to a stream. */
531 stream_printf(struct stream *stream, const char *fmt, ...)
540 ret = vsnprintf(buf->buf + buf->off + buf->in, buf_avail(buf), fmt, ap);
544 if ((unsigned)ret >= buf_avail(buf)) {
545 if ((unsigned)ret >= buf_size(buf))
546 buf_grow(buf, ret + 1);
547 if ((unsigned)ret >= buf_avail(buf)) {
548 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
558 /* Flush the entire write buffer of the stream. */
560 stream_flush(struct stream *stream)
564 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
568 /* Internal flush API. */
570 stream_flush_int(struct stream *stream, stream_flush_t how)
576 error = (*stream->filter->flushfn)(stream, buf, how);
577 assert(buf_count(buf) == 0);
581 /* The default flush method. */
583 stream_flush_default(struct stream *stream, struct buf *buf,
584 stream_flush_t __unused how)
588 while (buf_count(buf) > 0) {
590 n = (*stream->writefn)(stream->cookie,
591 buf->buf + buf->off, buf_count(buf));
592 } while (n == -1 && errno == EINTR);
600 /* Flush the write buffer and call fsync() on the file descriptor. */
602 stream_sync(struct stream *stream)
606 if (stream->fd == -1) {
610 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
613 error = fsync(stream->fd);
617 /* Like truncate() but on a stream. */
619 stream_truncate(struct stream *stream, off_t size)
623 if (stream->fd == -1) {
627 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
630 error = ftruncate(stream->fd, size);
634 /* Like stream_truncate() except the off_t parameter is an offset. */
636 stream_truncate_rel(struct stream *stream, off_t off)
641 if (stream->fd == -1) {
645 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
648 error = fstat(stream->fd, &sb);
651 error = stream_truncate(stream, sb.st_size + off);
655 /* Rewind the stream. */
657 stream_rewind(struct stream *stream)
661 if (stream->fd == -1) {
665 if (stream->rdbuf != NULL)
666 buf_less(stream->rdbuf, buf_count(stream->rdbuf));
667 if (stream->wrbuf != NULL) {
668 error = stream_flush_int(stream, STREAM_FLUSH_NORMAL);
672 error = lseek(stream->fd, 0, SEEK_SET);
676 /* Return EOF status. */
678 stream_eof(struct stream *stream)
681 return (stream->eof);
684 /* Close a stream and free any resources held by it. */
686 stream_close(struct stream *stream)
694 if (stream->wrbuf != NULL)
695 error = stream_flush_int(stream, STREAM_FLUSH_CLOSING);
696 stream_filter_fini(stream);
697 if (stream->closefn != NULL)
699 * We might overwrite a previous error from stream_flush(),
700 * but we have no choice, because wether it had worked or
701 * not, we need to close the file descriptor.
703 error = (*stream->closefn)(stream->cookie);
704 if (stream->rdbuf != NULL)
705 buf_free(stream->rdbuf);
706 if (stream->wrbuf != NULL)
707 buf_free(stream->wrbuf);
712 /* The default fill method. */
714 stream_fill_default(struct stream *stream, struct buf *buf)
720 assert(buf_avail(buf) > 0);
721 n = (*stream->readfn)(stream->cookie, buf->buf + buf->off + buf->in,
734 * Refill the read buffer. This function is not permitted to return
735 * without having made more bytes available, unless there was an error.
736 * Moreover, stream_fill() returns the number of bytes added.
739 stream_fill(struct stream *stream)
741 struct stream_filter *filter;
748 filter = stream->filter;
752 oldcount = buf_count(buf);
754 n = (*filter->fillfn)(stream, buf);
755 assert((n > 0 && n == (signed)(buf_count(buf) - oldcount)) ||
756 (n <= 0 && buf_count(buf) == oldcount));
761 * Lookup a stream filter.
763 * We are not supposed to get passed an invalid filter id, since
764 * filter ids are an enum type and we don't have invalid filter
765 * ids in the enum :-). Thus, we are not checking for out of
766 * bounds access here. If it happens, it's the caller's fault
769 static struct stream_filter *
770 stream_filter_lookup(stream_filter_t id)
772 struct stream_filter *filter;
774 filter = stream_filters;
775 while (filter->id != id)
781 stream_filter_init(struct stream *stream, void *data)
783 struct stream_filter *filter;
786 filter = stream->filter;
787 if (filter->initfn == NULL)
789 error = (*filter->initfn)(stream, data);
794 stream_filter_fini(struct stream *stream)
796 struct stream_filter *filter;
798 filter = stream->filter;
799 if (filter->finifn != NULL)
800 (*filter->finifn)(stream);
804 * Start a filter on a stream.
807 stream_filter_start(struct stream *stream, stream_filter_t id, void *data)
809 struct stream_filter *filter;
812 filter = stream->filter;
813 if (id == filter->id)
815 stream_filter_fini(stream);
816 stream->filter = stream_filter_lookup(id);
817 stream->fdata = NULL;
818 error = stream_filter_init(stream, data);
823 /* Stop a filter, this is equivalent to setting the null filter. */
825 stream_filter_stop(struct stream *stream)
828 stream_filter_start(stream, STREAM_FILTER_NULL, NULL);
831 /* The zlib stream filter implementation. */
833 /* Take no chances with zlib... */
835 zfilter_alloc(void __unused *opaque, unsigned int items, unsigned int size)
838 return (xmalloc(items * size));
842 zfilter_free(void __unused *opaque, void *ptr)
849 zfilter_init(struct stream *stream, void __unused *data)
856 zf = xmalloc(sizeof(struct zfilter));
857 memset(zf, 0, sizeof(struct zfilter));
858 if (stream->rdbuf != NULL) {
859 state = xmalloc(sizeof(z_stream));
860 state->zalloc = zfilter_alloc;
861 state->zfree = zfilter_free;
862 state->opaque = Z_NULL;
863 rv = inflateInit(state);
865 errx(1, "inflateInit: %s", state->msg);
866 buf = buf_new(buf_size(stream->rdbuf));
867 zf->rdbuf = stream->rdbuf;
871 if (stream->wrbuf != NULL) {
872 state = xmalloc(sizeof(z_stream));
873 state->zalloc = zfilter_alloc;
874 state->zfree = zfilter_free;
875 state->opaque = Z_NULL;
876 rv = deflateInit(state, Z_DEFAULT_COMPRESSION);
878 errx(1, "deflateInit: %s", state->msg);
879 buf = buf_new(buf_size(stream->wrbuf));
880 zf->wrbuf = stream->wrbuf;
889 zfilter_fini(struct stream *stream)
897 if (zf->rdbuf != NULL) {
901 * Even if it has produced all the bytes, zlib sometimes
902 * hasn't seen the EOF marker, so we need to call inflate()
903 * again to make sure we have eaten all the zlib'ed bytes.
905 if ((zf->flags & ZFILTER_EOF) == 0) {
906 n = zfilter_fill(stream, stream->rdbuf);
907 assert(n == 0 && zf->flags & ZFILTER_EOF);
911 buf_free(stream->rdbuf);
912 stream->rdbuf = zbuf;
914 if (zf->wrbuf != NULL) {
918 * Compress the remaining bytes in the buffer, if any,
919 * and emit an EOF marker as appropriate. We ignore
920 * the error because we can't do anything about it at
921 * this point, and it can happen if we're getting
924 (void)zfilter_flush(stream, stream->wrbuf,
925 STREAM_FLUSH_CLOSING);
928 buf_free(stream->wrbuf);
929 stream->wrbuf = zbuf;
935 zfilter_flush(struct stream *stream, struct buf *buf, stream_flush_t how)
940 size_t lastin, lastout, ate, prod;
941 int done, error, flags, rv;
947 if (how == STREAM_FLUSH_NORMAL)
948 flags = Z_SYNC_FLUSH;
957 * According to zlib.h, we should have at least 6 bytes
958 * available when using deflate() with Z_SYNC_FLUSH.
960 if ((buf_avail(zbuf) < 6 && flags == Z_SYNC_FLUSH) ||
961 rv == Z_BUF_ERROR || buf_avail(buf) == 0) {
962 error = stream_flush_default(stream, zbuf, how);
967 state->next_in = (Bytef *)(buf->buf + buf->off);
968 state->avail_in = buf_count(buf);
969 state->next_out = (Bytef *)(zbuf->buf + zbuf->off + zbuf->in);
970 state->avail_out = buf_avail(zbuf);
971 lastin = state->avail_in;
972 lastout = state->avail_out;
973 rv = deflate(state, flags);
974 if (rv != Z_BUF_ERROR && rv != Z_OK && rv != Z_STREAM_END)
975 errx(1, "deflate: %s", state->msg);
976 ate = lastin - state->avail_in;
977 prod = lastout - state->avail_out;
979 buf_more(zbuf, prod);
980 if ((flags == Z_SYNC_FLUSH && buf_count(buf) > 0) ||
981 (flags == Z_FINISH && rv != Z_STREAM_END) ||
985 assert(rv == Z_OK || (rv == Z_STREAM_END && flags == Z_FINISH));
986 error = stream_flush_default(stream, zbuf, how);
991 zfilter_fill(struct stream *stream, struct buf *buf)
996 size_t lastin, lastout, new;
1001 state = zf->rdstate;
1004 assert(buf_avail(buf) > 0);
1005 if (buf_count(zbuf) == 0) {
1006 n = stream_fill_default(stream, zbuf);
1011 assert(buf_count(zbuf) > 0);
1012 state->next_in = (Bytef *)(zbuf->buf + zbuf->off);
1013 state->avail_in = buf_count(zbuf);
1014 state->next_out = (Bytef *)(buf->buf + buf->off + buf->in);
1015 state->avail_out = buf_avail(buf);
1016 lastin = state->avail_in;
1017 lastout = state->avail_out;
1018 rv = inflate(state, Z_SYNC_FLUSH);
1019 buf_less(zbuf, lastin - state->avail_in);
1020 new = lastout - state->avail_out;
1021 if (new == 0 && rv != Z_STREAM_END) {
1022 n = stream_fill_default(stream, zbuf);
1029 if (rv != Z_STREAM_END && rv != Z_OK)
1030 errx(1, "inflate: %s", state->msg);
1031 if (rv == Z_STREAM_END)
1032 zf->flags |= ZFILTER_EOF;
1037 /* The MD5 stream filter implementation. */
1039 md5filter_init(struct stream *stream, void *data)
1041 struct md5filter *mf;
1043 mf = xmalloc(sizeof(struct md5filter));
1051 md5filter_fini(struct stream *stream)
1053 struct md5filter *mf;
1056 MD5_End(mf->md5, &mf->ctx);
1057 free(stream->fdata);
1061 md5filter_fill(struct stream *stream, struct buf *buf)
1065 assert(buf_avail(buf) > 0);
1066 n = stream_fill_default(stream, buf);
1071 md5filter_flush(struct stream *stream, struct buf *buf, stream_flush_t how)
1073 struct md5filter *mf;
1077 MD5_Update(&mf->ctx, buf->buf + buf->off, buf->in);
1078 error = stream_flush_default(stream, buf, how);