2 * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu>
3 * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include "event2/event-config.h"
29 #include "evconfig-private.h"
37 #ifdef EVENT__HAVE_VASPRINTF
38 /* If we have vasprintf, we need to define _GNU_SOURCE before we include
39 * stdio.h. This comes from evconfig-private.h.
43 #include <sys/types.h>
45 #ifdef EVENT__HAVE_SYS_TIME_H
49 #ifdef EVENT__HAVE_SYS_SOCKET_H
50 #include <sys/socket.h>
53 #ifdef EVENT__HAVE_SYS_UIO_H
57 #ifdef EVENT__HAVE_SYS_IOCTL_H
58 #include <sys/ioctl.h>
61 #ifdef EVENT__HAVE_SYS_MMAN_H
65 #ifdef EVENT__HAVE_SYS_SENDFILE_H
66 #include <sys/sendfile.h>
68 #ifdef EVENT__HAVE_SYS_STAT_H
77 #ifdef EVENT__HAVE_STDARG_H
80 #ifdef EVENT__HAVE_UNISTD_H
85 #include "event2/event.h"
86 #include "event2/buffer.h"
87 #include "event2/buffer_compat.h"
88 #include "event2/bufferevent.h"
89 #include "event2/bufferevent_compat.h"
90 #include "event2/bufferevent_struct.h"
91 #include "event2/thread.h"
92 #include "log-internal.h"
93 #include "mm-internal.h"
94 #include "util-internal.h"
95 #include "evthread-internal.h"
96 #include "evbuffer-internal.h"
97 #include "bufferevent-internal.h"
99 /* some systems do not have MAP_FAILED */
101 #define MAP_FAILED ((void *)-1)
104 /* send file support */
105 #if defined(EVENT__HAVE_SYS_SENDFILE_H) && defined(EVENT__HAVE_SENDFILE) && defined(__linux__)
106 #define USE_SENDFILE 1
107 #define SENDFILE_IS_LINUX 1
108 #elif defined(EVENT__HAVE_SENDFILE) && defined(__FreeBSD__)
109 #define USE_SENDFILE 1
110 #define SENDFILE_IS_FREEBSD 1
111 #elif defined(EVENT__HAVE_SENDFILE) && defined(__APPLE__)
112 #define USE_SENDFILE 1
113 #define SENDFILE_IS_MACOSX 1
114 #elif defined(EVENT__HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__)
115 #define USE_SENDFILE 1
116 #define SENDFILE_IS_SOLARIS 1
119 /* Mask of user-selectable callback flags. */
120 #define EVBUFFER_CB_USER_FLAGS 0xffff
121 /* Mask of all internal-use-only flags. */
122 #define EVBUFFER_CB_INTERNAL_FLAGS 0xffff0000
124 /* Flag set if the callback is using the cb_obsolete function pointer */
125 #define EVBUFFER_CB_OBSOLETE 0x00040000
127 /* evbuffer_chain support */
128 #define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off)
129 #define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \
130 0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off))
132 #define CHAIN_PINNED(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0)
133 #define CHAIN_PINNED_R(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0)
135 /* evbuffer_ptr support */
136 #define PTR_NOT_FOUND(ptr) do { \
138 (ptr)->internal_.chain = NULL; \
139 (ptr)->internal_.pos_in_chain = 0; \
142 static void evbuffer_chain_align(struct evbuffer_chain *chain);
143 static int evbuffer_chain_should_realign(struct evbuffer_chain *chain,
145 static void evbuffer_deferred_callback(struct event_callback *cb, void *arg);
146 static int evbuffer_ptr_memcmp(const struct evbuffer *buf,
147 const struct evbuffer_ptr *pos, const char *mem, size_t len);
148 static struct evbuffer_chain *evbuffer_expand_singlechain(struct evbuffer *buf,
150 static int evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos,
152 static int evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg);
153 static inline void evbuffer_chain_incref(struct evbuffer_chain *chain);
155 static struct evbuffer_chain *
156 evbuffer_chain_new(size_t size)
158 struct evbuffer_chain *chain;
161 if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE)
164 size += EVBUFFER_CHAIN_SIZE;
166 /* get the next largest memory that can hold the buffer */
167 if (size < EVBUFFER_CHAIN_MAX / 2) {
168 to_alloc = MIN_BUFFER_SIZE;
169 while (to_alloc < size) {
176 /* we get everything in one chunk */
177 if ((chain = mm_malloc(to_alloc)) == NULL)
180 memset(chain, 0, EVBUFFER_CHAIN_SIZE);
182 chain->buffer_len = to_alloc - EVBUFFER_CHAIN_SIZE;
184 /* this way we can manipulate the buffer to different addresses,
185 * which is required for mmap for example.
187 chain->buffer = EVBUFFER_CHAIN_EXTRA(u_char, chain);
195 evbuffer_chain_free(struct evbuffer_chain *chain)
197 EVUTIL_ASSERT(chain->refcnt > 0);
198 if (--chain->refcnt > 0) {
199 /* chain is still referenced by other chains */
203 if (CHAIN_PINNED(chain)) {
204 /* will get freed once no longer dangling */
206 chain->flags |= EVBUFFER_DANGLING;
210 /* safe to release chain, it's either a referencing
211 * chain or all references to it have been freed */
212 if (chain->flags & EVBUFFER_REFERENCE) {
213 struct evbuffer_chain_reference *info =
214 EVBUFFER_CHAIN_EXTRA(
215 struct evbuffer_chain_reference,
218 (*info->cleanupfn)(chain->buffer,
222 if (chain->flags & EVBUFFER_FILESEGMENT) {
223 struct evbuffer_chain_file_segment *info =
224 EVBUFFER_CHAIN_EXTRA(
225 struct evbuffer_chain_file_segment,
229 if (info->segment->is_mapping)
230 UnmapViewOfFile(chain->buffer);
232 evbuffer_file_segment_free(info->segment);
235 if (chain->flags & EVBUFFER_MULTICAST) {
236 struct evbuffer_multicast_parent *info =
237 EVBUFFER_CHAIN_EXTRA(
238 struct evbuffer_multicast_parent,
240 /* referencing chain is being freed, decrease
241 * refcounts of source chain and associated
242 * evbuffer (which get freed once both reach
244 EVUTIL_ASSERT(info->source != NULL);
245 EVUTIL_ASSERT(info->parent != NULL);
246 EVBUFFER_LOCK(info->source);
247 evbuffer_chain_free(info->parent);
248 evbuffer_decref_and_unlock_(info->source);
255 evbuffer_free_all_chains(struct evbuffer_chain *chain)
257 struct evbuffer_chain *next;
258 for (; chain; chain = next) {
260 evbuffer_chain_free(chain);
266 evbuffer_chains_all_empty(struct evbuffer_chain *chain)
268 for (; chain; chain = chain->next) {
275 /* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid
276 "unused variable" warnings. */
277 static inline int evbuffer_chains_all_empty(struct evbuffer_chain *chain) {
282 /* Free all trailing chains in 'buf' that are neither pinned nor empty, prior
283 * to replacing them all with a new chain. Return a pointer to the place
284 * where the new chain will go.
286 * Internal; requires lock. The caller must fix up buf->last and buf->first
287 * as needed; they might have been freed.
289 static struct evbuffer_chain **
290 evbuffer_free_trailing_empty_chains(struct evbuffer *buf)
292 struct evbuffer_chain **ch = buf->last_with_datap;
293 /* Find the first victim chain. It might be *last_with_datap */
294 while ((*ch) && ((*ch)->off != 0 || CHAIN_PINNED(*ch)))
297 EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch));
298 evbuffer_free_all_chains(*ch);
304 /* Add a single chain 'chain' to the end of 'buf', freeing trailing empty
305 * chains as necessary. Requires lock. Does not schedule callbacks.
308 evbuffer_chain_insert(struct evbuffer *buf,
309 struct evbuffer_chain *chain)
311 ASSERT_EVBUFFER_LOCKED(buf);
312 if (*buf->last_with_datap == NULL) {
313 /* There are no chains data on the buffer at all. */
314 EVUTIL_ASSERT(buf->last_with_datap == &buf->first);
315 EVUTIL_ASSERT(buf->first == NULL);
316 buf->first = buf->last = chain;
318 struct evbuffer_chain **chp;
319 chp = evbuffer_free_trailing_empty_chains(buf);
322 buf->last_with_datap = chp;
325 buf->total_len += chain->off;
328 static inline struct evbuffer_chain *
329 evbuffer_chain_insert_new(struct evbuffer *buf, size_t datlen)
331 struct evbuffer_chain *chain;
332 if ((chain = evbuffer_chain_new(datlen)) == NULL)
334 evbuffer_chain_insert(buf, chain);
339 evbuffer_chain_pin_(struct evbuffer_chain *chain, unsigned flag)
341 EVUTIL_ASSERT((chain->flags & flag) == 0);
342 chain->flags |= flag;
346 evbuffer_chain_unpin_(struct evbuffer_chain *chain, unsigned flag)
348 EVUTIL_ASSERT((chain->flags & flag) != 0);
349 chain->flags &= ~flag;
350 if (chain->flags & EVBUFFER_DANGLING)
351 evbuffer_chain_free(chain);
355 evbuffer_chain_incref(struct evbuffer_chain *chain)
363 struct evbuffer *buffer;
365 buffer = mm_calloc(1, sizeof(struct evbuffer));
369 LIST_INIT(&buffer->callbacks);
371 buffer->last_with_datap = &buffer->first;
377 evbuffer_set_flags(struct evbuffer *buf, ev_uint64_t flags)
380 buf->flags |= (ev_uint32_t)flags;
381 EVBUFFER_UNLOCK(buf);
386 evbuffer_clear_flags(struct evbuffer *buf, ev_uint64_t flags)
389 buf->flags &= ~(ev_uint32_t)flags;
390 EVBUFFER_UNLOCK(buf);
395 evbuffer_incref_(struct evbuffer *buf)
399 EVBUFFER_UNLOCK(buf);
403 evbuffer_incref_and_lock_(struct evbuffer *buf)
410 evbuffer_defer_callbacks(struct evbuffer *buffer, struct event_base *base)
412 EVBUFFER_LOCK(buffer);
413 buffer->cb_queue = base;
414 buffer->deferred_cbs = 1;
415 event_deferred_cb_init_(&buffer->deferred,
416 event_base_get_npriorities(base) / 2,
417 evbuffer_deferred_callback, buffer);
418 EVBUFFER_UNLOCK(buffer);
423 evbuffer_enable_locking(struct evbuffer *buf, void *lock)
425 #ifdef EVENT__DISABLE_THREAD_SUPPORT
432 EVTHREAD_ALLOC_LOCK(lock, EVTHREAD_LOCKTYPE_RECURSIVE);
447 evbuffer_set_parent_(struct evbuffer *buf, struct bufferevent *bev)
451 EVBUFFER_UNLOCK(buf);
455 evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred)
457 struct evbuffer_cb_entry *cbent, *next;
458 struct evbuffer_cb_info info;
460 ev_uint32_t mask, masked_val;
463 if (running_deferred) {
464 mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
465 masked_val = EVBUFFER_CB_ENABLED;
466 } else if (buffer->deferred_cbs) {
467 mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
468 masked_val = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
469 /* Don't zero-out n_add/n_del, since the deferred callbacks
470 will want to see them. */
473 mask = EVBUFFER_CB_ENABLED;
474 masked_val = EVBUFFER_CB_ENABLED;
477 ASSERT_EVBUFFER_LOCKED(buffer);
479 if (LIST_EMPTY(&buffer->callbacks)) {
480 buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
483 if (buffer->n_add_for_cb == 0 && buffer->n_del_for_cb == 0)
486 new_size = buffer->total_len;
487 info.orig_size = new_size + buffer->n_del_for_cb - buffer->n_add_for_cb;
488 info.n_added = buffer->n_add_for_cb;
489 info.n_deleted = buffer->n_del_for_cb;
491 buffer->n_add_for_cb = 0;
492 buffer->n_del_for_cb = 0;
494 for (cbent = LIST_FIRST(&buffer->callbacks);
495 cbent != LIST_END(&buffer->callbacks);
497 /* Get the 'next' pointer now in case this callback decides
498 * to remove itself or something. */
499 next = LIST_NEXT(cbent, next);
501 if ((cbent->flags & mask) != masked_val)
504 if ((cbent->flags & EVBUFFER_CB_OBSOLETE))
505 cbent->cb.cb_obsolete(buffer,
506 info.orig_size, new_size, cbent->cbarg);
508 cbent->cb.cb_func(buffer, &info, cbent->cbarg);
513 evbuffer_invoke_callbacks_(struct evbuffer *buffer)
515 if (LIST_EMPTY(&buffer->callbacks)) {
516 buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
520 if (buffer->deferred_cbs) {
521 if (event_deferred_cb_schedule_(buffer->cb_queue, &buffer->deferred)) {
522 evbuffer_incref_and_lock_(buffer);
524 bufferevent_incref_(buffer->parent);
526 EVBUFFER_UNLOCK(buffer);
529 evbuffer_run_callbacks(buffer, 0);
533 evbuffer_deferred_callback(struct event_callback *cb, void *arg)
535 struct bufferevent *parent = NULL;
536 struct evbuffer *buffer = arg;
538 /* XXXX It would be better to run these callbacks without holding the
540 EVBUFFER_LOCK(buffer);
541 parent = buffer->parent;
542 evbuffer_run_callbacks(buffer, 1);
543 evbuffer_decref_and_unlock_(buffer);
545 bufferevent_decref_(parent);
549 evbuffer_remove_all_callbacks(struct evbuffer *buffer)
551 struct evbuffer_cb_entry *cbent;
553 while ((cbent = LIST_FIRST(&buffer->callbacks))) {
554 LIST_REMOVE(cbent, next);
560 evbuffer_decref_and_unlock_(struct evbuffer *buffer)
562 struct evbuffer_chain *chain, *next;
563 ASSERT_EVBUFFER_LOCKED(buffer);
565 EVUTIL_ASSERT(buffer->refcnt > 0);
567 if (--buffer->refcnt > 0) {
568 EVBUFFER_UNLOCK(buffer);
572 for (chain = buffer->first; chain != NULL; chain = next) {
574 evbuffer_chain_free(chain);
576 evbuffer_remove_all_callbacks(buffer);
577 if (buffer->deferred_cbs)
578 event_deferred_cb_cancel_(buffer->cb_queue, &buffer->deferred);
580 EVBUFFER_UNLOCK(buffer);
581 if (buffer->own_lock)
582 EVTHREAD_FREE_LOCK(buffer->lock, EVTHREAD_LOCKTYPE_RECURSIVE);
587 evbuffer_free(struct evbuffer *buffer)
589 EVBUFFER_LOCK(buffer);
590 evbuffer_decref_and_unlock_(buffer);
594 evbuffer_lock(struct evbuffer *buf)
600 evbuffer_unlock(struct evbuffer *buf)
602 EVBUFFER_UNLOCK(buf);
606 evbuffer_get_length(const struct evbuffer *buffer)
610 EVBUFFER_LOCK(buffer);
612 result = (buffer->total_len);
614 EVBUFFER_UNLOCK(buffer);
620 evbuffer_get_contiguous_space(const struct evbuffer *buf)
622 struct evbuffer_chain *chain;
627 result = (chain != NULL ? chain->off : 0);
628 EVBUFFER_UNLOCK(buf);
634 evbuffer_add_iovec(struct evbuffer * buf, struct evbuffer_iovec * vec, int n_vec) {
643 for (n = 0; n < n_vec; n++) {
644 to_alloc += vec[n].iov_len;
647 if (evbuffer_expand_fast_(buf, to_alloc, 2) < 0) {
651 for (n = 0; n < n_vec; n++) {
652 /* XXX each 'add' call here does a bunch of setup that's
653 * obviated by evbuffer_expand_fast_, and some cleanup that we
654 * would like to do only once. Instead we should just extract
655 * the part of the code that's needed. */
657 if (evbuffer_add(buf, vec[n].iov_base, vec[n].iov_len) < 0) {
661 res += vec[n].iov_len;
665 EVBUFFER_UNLOCK(buf);
670 evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size,
671 struct evbuffer_iovec *vec, int n_vecs)
673 struct evbuffer_chain *chain, **chainp;
682 if ((chain = evbuffer_expand_singlechain(buf, size)) == NULL)
685 vec[0].iov_base = CHAIN_SPACE_PTR(chain);
686 vec[0].iov_len = (size_t) CHAIN_SPACE_LEN(chain);
687 EVUTIL_ASSERT(size<0 || (size_t)vec[0].iov_len >= (size_t)size);
690 if (evbuffer_expand_fast_(buf, size, n_vecs)<0)
692 n = evbuffer_read_setup_vecs_(buf, size, vec, n_vecs,
697 EVBUFFER_UNLOCK(buf);
703 advance_last_with_data(struct evbuffer *buf)
706 ASSERT_EVBUFFER_LOCKED(buf);
708 if (!*buf->last_with_datap)
711 while ((*buf->last_with_datap)->next && (*buf->last_with_datap)->next->off) {
712 buf->last_with_datap = &(*buf->last_with_datap)->next;
719 evbuffer_commit_space(struct evbuffer *buf,
720 struct evbuffer_iovec *vec, int n_vecs)
722 struct evbuffer_chain *chain, **firstchainp, **chainp;
734 } else if (n_vecs == 1 &&
735 (buf->last && vec[0].iov_base == (void*)CHAIN_SPACE_PTR(buf->last))) {
736 /* The user only got or used one chain; it might not
737 * be the first one with space in it. */
738 if ((size_t)vec[0].iov_len > (size_t)CHAIN_SPACE_LEN(buf->last))
740 buf->last->off += vec[0].iov_len;
741 added = vec[0].iov_len;
743 advance_last_with_data(buf);
747 /* Advance 'firstchain' to the first chain with space in it. */
748 firstchainp = buf->last_with_datap;
751 if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
752 firstchainp = &(*firstchainp)->next;
755 chain = *firstchainp;
756 /* pass 1: make sure that the pointers and lengths of vecs[] are in
757 * bounds before we try to commit anything. */
758 for (i=0; i<n_vecs; ++i) {
761 if (vec[i].iov_base != (void*)CHAIN_SPACE_PTR(chain) ||
762 (size_t)vec[i].iov_len > CHAIN_SPACE_LEN(chain))
766 /* pass 2: actually adjust all the chains. */
767 chainp = firstchainp;
768 for (i=0; i<n_vecs; ++i) {
769 (*chainp)->off += vec[i].iov_len;
770 added += vec[i].iov_len;
771 if (vec[i].iov_len) {
772 buf->last_with_datap = chainp;
774 chainp = &(*chainp)->next;
778 buf->total_len += added;
779 buf->n_add_for_cb += added;
781 evbuffer_invoke_callbacks_(buf);
784 EVBUFFER_UNLOCK(buf);
789 HAS_PINNED_R(struct evbuffer *buf)
791 return (buf->last && CHAIN_PINNED_R(buf->last));
795 ZERO_CHAIN(struct evbuffer *dst)
797 ASSERT_EVBUFFER_LOCKED(dst);
800 dst->last_with_datap = &(dst)->first;
804 /* Prepares the contents of src to be moved to another buffer by removing
805 * read-pinned chains. The first pinned chain is saved in first, and the
806 * last in last. If src has no read-pinned chains, first and last are set
809 PRESERVE_PINNED(struct evbuffer *src, struct evbuffer_chain **first,
810 struct evbuffer_chain **last)
812 struct evbuffer_chain *chain, **pinned;
814 ASSERT_EVBUFFER_LOCKED(src);
816 if (!HAS_PINNED_R(src)) {
817 *first = *last = NULL;
821 pinned = src->last_with_datap;
822 if (!CHAIN_PINNED_R(*pinned))
823 pinned = &(*pinned)->next;
824 EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned));
825 chain = *first = *pinned;
828 /* If there's data in the first pinned chain, we need to allocate
829 * a new chain and copy the data over. */
831 struct evbuffer_chain *tmp;
833 EVUTIL_ASSERT(pinned == src->last_with_datap);
834 tmp = evbuffer_chain_new(chain->off);
837 memcpy(tmp->buffer, chain->buffer + chain->misalign,
839 tmp->off = chain->off;
840 *src->last_with_datap = tmp;
842 chain->misalign += chain->off;
845 src->last = *src->last_with_datap;
853 RESTORE_PINNED(struct evbuffer *src, struct evbuffer_chain *pinned,
854 struct evbuffer_chain *last)
856 ASSERT_EVBUFFER_LOCKED(src);
865 src->last_with_datap = &src->first;
870 COPY_CHAIN(struct evbuffer *dst, struct evbuffer *src)
872 ASSERT_EVBUFFER_LOCKED(dst);
873 ASSERT_EVBUFFER_LOCKED(src);
874 dst->first = src->first;
875 if (src->last_with_datap == &src->first)
876 dst->last_with_datap = &dst->first;
878 dst->last_with_datap = src->last_with_datap;
879 dst->last = src->last;
880 dst->total_len = src->total_len;
884 APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
886 ASSERT_EVBUFFER_LOCKED(dst);
887 ASSERT_EVBUFFER_LOCKED(src);
888 dst->last->next = src->first;
889 if (src->last_with_datap == &src->first)
890 dst->last_with_datap = &dst->last->next;
892 dst->last_with_datap = src->last_with_datap;
893 dst->last = src->last;
894 dst->total_len += src->total_len;
898 APPEND_CHAIN_MULTICAST(struct evbuffer *dst, struct evbuffer *src)
900 struct evbuffer_chain *tmp;
901 struct evbuffer_chain *chain = src->first;
902 struct evbuffer_multicast_parent *extra;
904 ASSERT_EVBUFFER_LOCKED(dst);
905 ASSERT_EVBUFFER_LOCKED(src);
907 for (; chain; chain = chain->next) {
908 if (!chain->off || chain->flags & EVBUFFER_DANGLING) {
909 /* skip empty chains */
913 tmp = evbuffer_chain_new(sizeof(struct evbuffer_multicast_parent));
915 event_warn("%s: out of memory", __func__);
918 extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_multicast_parent, tmp);
919 /* reference evbuffer containing source chain so it
920 * doesn't get released while the chain is still
921 * being referenced to */
922 evbuffer_incref_(src);
924 /* reference source chain which now becomes immutable */
925 evbuffer_chain_incref(chain);
926 extra->parent = chain;
927 chain->flags |= EVBUFFER_IMMUTABLE;
928 tmp->buffer_len = chain->buffer_len;
929 tmp->misalign = chain->misalign;
930 tmp->off = chain->off;
931 tmp->flags |= EVBUFFER_MULTICAST|EVBUFFER_IMMUTABLE;
932 tmp->buffer = chain->buffer;
933 evbuffer_chain_insert(dst, tmp);
938 PREPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
940 ASSERT_EVBUFFER_LOCKED(dst);
941 ASSERT_EVBUFFER_LOCKED(src);
942 src->last->next = dst->first;
943 dst->first = src->first;
944 dst->total_len += src->total_len;
945 if (*dst->last_with_datap == NULL) {
946 if (src->last_with_datap == &(src)->first)
947 dst->last_with_datap = &dst->first;
949 dst->last_with_datap = src->last_with_datap;
950 } else if (dst->last_with_datap == &dst->first) {
951 dst->last_with_datap = &src->last->next;
956 evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
958 struct evbuffer_chain *pinned, *last;
959 size_t in_total_len, out_total_len;
962 EVBUFFER_LOCK2(inbuf, outbuf);
963 in_total_len = inbuf->total_len;
964 out_total_len = outbuf->total_len;
966 if (in_total_len == 0 || outbuf == inbuf)
969 if (outbuf->freeze_end || inbuf->freeze_start) {
974 if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
979 if (out_total_len == 0) {
980 /* There might be an empty chain at the start of outbuf; free
982 evbuffer_free_all_chains(outbuf->first);
983 COPY_CHAIN(outbuf, inbuf);
985 APPEND_CHAIN(outbuf, inbuf);
988 RESTORE_PINNED(inbuf, pinned, last);
990 inbuf->n_del_for_cb += in_total_len;
991 outbuf->n_add_for_cb += in_total_len;
993 evbuffer_invoke_callbacks_(inbuf);
994 evbuffer_invoke_callbacks_(outbuf);
997 EVBUFFER_UNLOCK2(inbuf, outbuf);
1002 evbuffer_add_buffer_reference(struct evbuffer *outbuf, struct evbuffer *inbuf)
1004 size_t in_total_len, out_total_len;
1005 struct evbuffer_chain *chain;
1008 EVBUFFER_LOCK2(inbuf, outbuf);
1009 in_total_len = inbuf->total_len;
1010 out_total_len = outbuf->total_len;
1011 chain = inbuf->first;
1013 if (in_total_len == 0)
1016 if (outbuf->freeze_end || outbuf == inbuf) {
1021 for (; chain; chain = chain->next) {
1022 if ((chain->flags & (EVBUFFER_FILESEGMENT|EVBUFFER_SENDFILE|EVBUFFER_MULTICAST)) != 0) {
1023 /* chain type can not be referenced */
1029 if (out_total_len == 0) {
1030 /* There might be an empty chain at the start of outbuf; free
1032 evbuffer_free_all_chains(outbuf->first);
1034 APPEND_CHAIN_MULTICAST(outbuf, inbuf);
1036 outbuf->n_add_for_cb += in_total_len;
1037 evbuffer_invoke_callbacks_(outbuf);
1040 EVBUFFER_UNLOCK2(inbuf, outbuf);
1045 evbuffer_prepend_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
1047 struct evbuffer_chain *pinned, *last;
1048 size_t in_total_len, out_total_len;
1051 EVBUFFER_LOCK2(inbuf, outbuf);
1053 in_total_len = inbuf->total_len;
1054 out_total_len = outbuf->total_len;
1056 if (!in_total_len || inbuf == outbuf)
1059 if (outbuf->freeze_start || inbuf->freeze_start) {
1064 if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
1069 if (out_total_len == 0) {
1070 /* There might be an empty chain at the start of outbuf; free
1072 evbuffer_free_all_chains(outbuf->first);
1073 COPY_CHAIN(outbuf, inbuf);
1075 PREPEND_CHAIN(outbuf, inbuf);
1078 RESTORE_PINNED(inbuf, pinned, last);
1080 inbuf->n_del_for_cb += in_total_len;
1081 outbuf->n_add_for_cb += in_total_len;
1083 evbuffer_invoke_callbacks_(inbuf);
1084 evbuffer_invoke_callbacks_(outbuf);
1086 EVBUFFER_UNLOCK2(inbuf, outbuf);
1091 evbuffer_drain(struct evbuffer *buf, size_t len)
1093 struct evbuffer_chain *chain, *next;
1094 size_t remaining, old_len;
1098 old_len = buf->total_len;
1103 if (buf->freeze_start) {
1108 if (len >= old_len && !HAS_PINNED_R(buf)) {
1110 for (chain = buf->first; chain != NULL; chain = next) {
1112 evbuffer_chain_free(chain);
1120 buf->total_len -= len;
1122 for (chain = buf->first;
1123 remaining >= chain->off;
1126 remaining -= chain->off;
1128 if (chain == *buf->last_with_datap) {
1129 buf->last_with_datap = &buf->first;
1131 if (&chain->next == buf->last_with_datap)
1132 buf->last_with_datap = &buf->first;
1134 if (CHAIN_PINNED_R(chain)) {
1135 EVUTIL_ASSERT(remaining == 0);
1136 chain->misalign += chain->off;
1140 evbuffer_chain_free(chain);
1144 EVUTIL_ASSERT(chain && remaining <= chain->off);
1145 chain->misalign += remaining;
1146 chain->off -= remaining;
1149 buf->n_del_for_cb += len;
1150 /* Tell someone about changes in this buffer */
1151 evbuffer_invoke_callbacks_(buf);
1154 EVBUFFER_UNLOCK(buf);
1158 /* Reads data from an event buffer and drains the bytes read */
1160 evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen)
1164 n = evbuffer_copyout_from(buf, NULL, data_out, datlen);
1166 if (evbuffer_drain(buf, n)<0)
1169 EVBUFFER_UNLOCK(buf);
1174 evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen)
1176 return evbuffer_copyout_from(buf, NULL, data_out, datlen);
1180 evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos,
1181 void *data_out, size_t datlen)
1183 /*XXX fails badly on sendfile case. */
1184 struct evbuffer_chain *chain;
1185 char *data = data_out;
1187 ev_ssize_t result = 0;
1188 size_t pos_in_chain;
1193 if (datlen > (size_t)(EV_SSIZE_MAX - pos->pos)) {
1197 chain = pos->internal_.chain;
1198 pos_in_chain = pos->internal_.pos_in_chain;
1199 if (datlen + pos->pos > buf->total_len)
1200 datlen = buf->total_len - pos->pos;
1204 if (datlen > buf->total_len)
1205 datlen = buf->total_len;
1212 if (buf->freeze_start) {
1219 while (datlen && datlen >= chain->off - pos_in_chain) {
1220 size_t copylen = chain->off - pos_in_chain;
1222 chain->buffer + chain->misalign + pos_in_chain,
1227 chain = chain->next;
1229 EVUTIL_ASSERT(chain || datlen==0);
1233 EVUTIL_ASSERT(chain);
1234 EVUTIL_ASSERT(datlen+pos_in_chain <= chain->off);
1236 memcpy(data, chain->buffer + chain->misalign + pos_in_chain,
1242 EVBUFFER_UNLOCK(buf);
1246 /* reads data from the src buffer to the dst buffer, avoids memcpy as
1248 /* XXXX should return ev_ssize_t */
1250 evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst,
1253 /*XXX We should have an option to force this to be zero-copy.*/
1255 /*XXX can fail badly on sendfile case. */
1256 struct evbuffer_chain *chain, *previous;
1260 EVBUFFER_LOCK2(src, dst);
1262 chain = previous = src->first;
1264 if (datlen == 0 || dst == src) {
1269 if (dst->freeze_end || src->freeze_start) {
1274 /* short-cut if there is no more data buffered */
1275 if (datlen >= src->total_len) {
1276 datlen = src->total_len;
1277 evbuffer_add_buffer(dst, src);
1278 result = (int)datlen; /*XXXX should return ev_ssize_t*/
1282 /* removes chains if possible */
1283 while (chain->off <= datlen) {
1284 /* We can't remove the last with data from src unless we
1285 * remove all chains, in which case we would have done the if
1287 EVUTIL_ASSERT(chain != *src->last_with_datap);
1288 nread += chain->off;
1289 datlen -= chain->off;
1291 if (src->last_with_datap == &chain->next)
1292 src->last_with_datap = &src->first;
1293 chain = chain->next;
1297 /* we can remove the chain */
1298 struct evbuffer_chain **chp;
1299 chp = evbuffer_free_trailing_empty_chains(dst);
1301 if (dst->first == NULL) {
1302 dst->first = src->first;
1306 dst->last = previous;
1307 previous->next = NULL;
1309 advance_last_with_data(dst);
1311 dst->total_len += nread;
1312 dst->n_add_for_cb += nread;
1315 /* we know that there is more data in the src buffer than
1316 * we want to read, so we manually drain the chain */
1317 evbuffer_add(dst, chain->buffer + chain->misalign, datlen);
1318 chain->misalign += datlen;
1319 chain->off -= datlen;
1322 /* You might think we would want to increment dst->n_add_for_cb
1323 * here too. But evbuffer_add above already took care of that.
1325 src->total_len -= nread;
1326 src->n_del_for_cb += nread;
1329 evbuffer_invoke_callbacks_(dst);
1330 evbuffer_invoke_callbacks_(src);
1332 result = (int)nread;/*XXXX should change return type */
1335 EVBUFFER_UNLOCK2(src, dst);
1340 evbuffer_pullup(struct evbuffer *buf, ev_ssize_t size)
1342 struct evbuffer_chain *chain, *next, *tmp, *last_with_data;
1343 unsigned char *buffer, *result = NULL;
1344 ev_ssize_t remaining;
1345 int removed_last_with_data = 0;
1346 int removed_last_with_datap = 0;
1353 size = buf->total_len;
1354 /* if size > buf->total_len, we cannot guarantee to the user that she
1355 * is going to have a long enough buffer afterwards; so we return
1357 if (size == 0 || (size_t)size > buf->total_len)
1360 /* No need to pull up anything; the first size bytes are
1362 if (chain->off >= (size_t)size) {
1363 result = chain->buffer + chain->misalign;
1367 /* Make sure that none of the chains we need to copy from is pinned. */
1368 remaining = size - chain->off;
1369 EVUTIL_ASSERT(remaining >= 0);
1370 for (tmp=chain->next; tmp; tmp=tmp->next) {
1371 if (CHAIN_PINNED(tmp))
1373 if (tmp->off >= (size_t)remaining)
1375 remaining -= tmp->off;
1378 if (CHAIN_PINNED(chain)) {
1379 size_t old_off = chain->off;
1380 if (CHAIN_SPACE_LEN(chain) < size - chain->off) {
1381 /* not enough room at end of chunk. */
1384 buffer = CHAIN_SPACE_PTR(chain);
1388 chain = chain->next;
1389 } else if (chain->buffer_len - chain->misalign >= (size_t)size) {
1390 /* already have enough space in the first chain */
1391 size_t old_off = chain->off;
1392 buffer = chain->buffer + chain->misalign + chain->off;
1396 chain = chain->next;
1398 if ((tmp = evbuffer_chain_new(size)) == NULL) {
1399 event_warn("%s: out of memory", __func__);
1402 buffer = tmp->buffer;
1407 /* TODO(niels): deal with buffers that point to NULL like sendfile */
1409 /* Copy and free every chunk that will be entirely pulled into tmp */
1410 last_with_data = *buf->last_with_datap;
1411 for (; chain != NULL && (size_t)size >= chain->off; chain = next) {
1414 memcpy(buffer, chain->buffer + chain->misalign, chain->off);
1416 buffer += chain->off;
1417 if (chain == last_with_data)
1418 removed_last_with_data = 1;
1419 if (&chain->next == buf->last_with_datap)
1420 removed_last_with_datap = 1;
1422 evbuffer_chain_free(chain);
1425 if (chain != NULL) {
1426 memcpy(buffer, chain->buffer + chain->misalign, size);
1427 chain->misalign += size;
1435 if (removed_last_with_data) {
1436 buf->last_with_datap = &buf->first;
1437 } else if (removed_last_with_datap) {
1438 if (buf->first->next && buf->first->next->off)
1439 buf->last_with_datap = &buf->first->next;
1441 buf->last_with_datap = &buf->first;
1444 result = (tmp->buffer + tmp->misalign);
1447 EVBUFFER_UNLOCK(buf);
1452 * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'.
1453 * The returned buffer needs to be freed by the called.
1456 evbuffer_readline(struct evbuffer *buffer)
1458 return evbuffer_readln(buffer, NULL, EVBUFFER_EOL_ANY);
1461 static inline ev_ssize_t
1462 evbuffer_strchr(struct evbuffer_ptr *it, const char chr)
1464 struct evbuffer_chain *chain = it->internal_.chain;
1465 size_t i = it->internal_.pos_in_chain;
1466 while (chain != NULL) {
1467 char *buffer = (char *)chain->buffer + chain->misalign;
1468 char *cp = memchr(buffer+i, chr, chain->off-i);
1470 it->internal_.chain = chain;
1471 it->internal_.pos_in_chain = cp - buffer;
1472 it->pos += (cp - buffer - i);
1475 it->pos += chain->off - i;
1477 chain = chain->next;
1483 static inline char *
1484 find_eol_char(char *s, size_t len)
1486 #define CHUNK_SZ 128
1487 /* Lots of benchmarking found this approach to be faster in practice
1488 * than doing two memchrs over the whole buffer, doin a memchr on each
1489 * char of the buffer, or trying to emulate memchr by hand. */
1490 char *s_end, *cr, *lf;
1493 size_t chunk = (s + CHUNK_SZ < s_end) ? CHUNK_SZ : (s_end - s);
1494 cr = memchr(s, '\r', chunk);
1495 lf = memchr(s, '\n', chunk);
1511 evbuffer_find_eol_char(struct evbuffer_ptr *it)
1513 struct evbuffer_chain *chain = it->internal_.chain;
1514 size_t i = it->internal_.pos_in_chain;
1515 while (chain != NULL) {
1516 char *buffer = (char *)chain->buffer + chain->misalign;
1517 char *cp = find_eol_char(buffer+i, chain->off-i);
1519 it->internal_.chain = chain;
1520 it->internal_.pos_in_chain = cp - buffer;
1521 it->pos += (cp - buffer) - i;
1524 it->pos += chain->off - i;
1526 chain = chain->next;
1534 struct evbuffer_ptr *ptr, const char *chrset)
1537 struct evbuffer_chain *chain = ptr->internal_.chain;
1538 size_t i = ptr->internal_.pos_in_chain;
1544 char *buffer = (char *)chain->buffer + chain->misalign;
1545 for (; i < chain->off; ++i) {
1546 const char *p = chrset;
1548 if (buffer[i] == *p++)
1551 ptr->internal_.chain = chain;
1552 ptr->internal_.pos_in_chain = i;
1560 if (! chain->next) {
1561 ptr->internal_.chain = chain;
1562 ptr->internal_.pos_in_chain = i;
1567 chain = chain->next;
1573 evbuffer_getchr(struct evbuffer_ptr *it)
1575 struct evbuffer_chain *chain = it->internal_.chain;
1576 size_t off = it->internal_.pos_in_chain;
1581 return (unsigned char)chain->buffer[chain->misalign + off];
1585 evbuffer_search_eol(struct evbuffer *buffer,
1586 struct evbuffer_ptr *start, size_t *eol_len_out,
1587 enum evbuffer_eol_style eol_style)
1589 struct evbuffer_ptr it, it2;
1590 size_t extra_drain = 0;
1593 /* Avoid locking in trivial edge cases */
1594 if (start && start->internal_.chain == NULL) {
1597 *eol_len_out = extra_drain;
1601 EVBUFFER_LOCK(buffer);
1604 memcpy(&it, start, sizeof(it));
1607 it.internal_.chain = buffer->first;
1608 it.internal_.pos_in_chain = 0;
1611 /* the eol_style determines our first stop character and how many
1612 * characters we are going to drain afterwards. */
1613 switch (eol_style) {
1614 case EVBUFFER_EOL_ANY:
1615 if (evbuffer_find_eol_char(&it) < 0)
1617 memcpy(&it2, &it, sizeof(it));
1618 extra_drain = evbuffer_strspn(&it2, "\r\n");
1620 case EVBUFFER_EOL_CRLF_STRICT: {
1621 it = evbuffer_search(buffer, "\r\n", 2, &it);
1627 case EVBUFFER_EOL_CRLF: {
1628 ev_ssize_t start_pos = it.pos;
1629 /* Look for a LF ... */
1630 if (evbuffer_strchr(&it, '\n') < 0)
1633 /* ... optionally preceeded by a CR. */
1634 if (it.pos == start_pos)
1635 break; /* If the first character is \n, don't back up */
1636 /* This potentially does an extra linear walk over the first
1637 * few chains. Probably, that's not too expensive unless you
1638 * have a really pathological setup. */
1639 memcpy(&it2, &it, sizeof(it));
1640 if (evbuffer_ptr_subtract(buffer, &it2, 1)<0)
1642 if (evbuffer_getchr(&it2) == '\r') {
1643 memcpy(&it, &it2, sizeof(it));
1648 case EVBUFFER_EOL_LF:
1649 if (evbuffer_strchr(&it, '\n') < 0)
1653 case EVBUFFER_EOL_NUL:
1654 if (evbuffer_strchr(&it, '\0') < 0)
1664 EVBUFFER_UNLOCK(buffer);
1669 *eol_len_out = extra_drain;
1675 evbuffer_readln(struct evbuffer *buffer, size_t *n_read_out,
1676 enum evbuffer_eol_style eol_style)
1678 struct evbuffer_ptr it;
1680 size_t n_to_copy=0, extra_drain=0;
1681 char *result = NULL;
1683 EVBUFFER_LOCK(buffer);
1685 if (buffer->freeze_start) {
1689 it = evbuffer_search_eol(buffer, NULL, &extra_drain, eol_style);
1694 if ((line = mm_malloc(n_to_copy+1)) == NULL) {
1695 event_warn("%s: out of memory", __func__);
1699 evbuffer_remove(buffer, line, n_to_copy);
1700 line[n_to_copy] = '\0';
1702 evbuffer_drain(buffer, extra_drain);
1705 EVBUFFER_UNLOCK(buffer);
1708 *n_read_out = result ? n_to_copy : 0;
1713 #define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096
1715 /* Adds data to an event buffer */
1718 evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen)
1720 struct evbuffer_chain *chain, *tmp;
1721 const unsigned char *data = data_in;
1722 size_t remain, to_alloc;
1727 if (buf->freeze_end) {
1730 /* Prevent buf->total_len overflow */
1731 if (datlen > EV_SIZE_MAX - buf->total_len) {
1737 /* If there are no chains allocated for this buffer, allocate one
1738 * big enough to hold all the data. */
1739 if (chain == NULL) {
1740 chain = evbuffer_chain_new(datlen);
1743 evbuffer_chain_insert(buf, chain);
1746 if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
1747 /* Always true for mutable buffers */
1748 EVUTIL_ASSERT(chain->misalign >= 0 &&
1749 (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX);
1750 remain = chain->buffer_len - (size_t)chain->misalign - chain->off;
1751 if (remain >= datlen) {
1752 /* there's enough space to hold all the data in the
1753 * current last chain */
1754 memcpy(chain->buffer + chain->misalign + chain->off,
1756 chain->off += datlen;
1757 buf->total_len += datlen;
1758 buf->n_add_for_cb += datlen;
1760 } else if (!CHAIN_PINNED(chain) &&
1761 evbuffer_chain_should_realign(chain, datlen)) {
1762 /* we can fit the data into the misalignment */
1763 evbuffer_chain_align(chain);
1765 memcpy(chain->buffer + chain->off, data, datlen);
1766 chain->off += datlen;
1767 buf->total_len += datlen;
1768 buf->n_add_for_cb += datlen;
1772 /* we cannot write any data to the last chain */
1776 /* we need to add another chain */
1777 to_alloc = chain->buffer_len;
1778 if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE/2)
1780 if (datlen > to_alloc)
1782 tmp = evbuffer_chain_new(to_alloc);
1787 memcpy(chain->buffer + chain->misalign + chain->off,
1789 chain->off += remain;
1790 buf->total_len += remain;
1791 buf->n_add_for_cb += remain;
1797 memcpy(tmp->buffer, data, datlen);
1799 evbuffer_chain_insert(buf, tmp);
1800 buf->n_add_for_cb += datlen;
1803 evbuffer_invoke_callbacks_(buf);
1806 EVBUFFER_UNLOCK(buf);
1811 evbuffer_prepend(struct evbuffer *buf, const void *data, size_t datlen)
1813 struct evbuffer_chain *chain, *tmp;
1818 if (buf->freeze_start) {
1821 if (datlen > EV_SIZE_MAX - buf->total_len) {
1827 if (chain == NULL) {
1828 chain = evbuffer_chain_new(datlen);
1831 evbuffer_chain_insert(buf, chain);
1834 /* we cannot touch immutable buffers */
1835 if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
1836 /* Always true for mutable buffers */
1837 EVUTIL_ASSERT(chain->misalign >= 0 &&
1838 (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX);
1840 /* If this chain is empty, we can treat it as
1841 * 'empty at the beginning' rather than 'empty at the end' */
1842 if (chain->off == 0)
1843 chain->misalign = chain->buffer_len;
1845 if ((size_t)chain->misalign >= datlen) {
1846 /* we have enough space to fit everything */
1847 memcpy(chain->buffer + chain->misalign - datlen,
1849 chain->off += datlen;
1850 chain->misalign -= datlen;
1851 buf->total_len += datlen;
1852 buf->n_add_for_cb += datlen;
1854 } else if (chain->misalign) {
1855 /* we can only fit some of the data. */
1856 memcpy(chain->buffer,
1857 (char*)data + datlen - chain->misalign,
1858 (size_t)chain->misalign);
1859 chain->off += (size_t)chain->misalign;
1860 buf->total_len += (size_t)chain->misalign;
1861 buf->n_add_for_cb += (size_t)chain->misalign;
1862 datlen -= (size_t)chain->misalign;
1863 chain->misalign = 0;
1867 /* we need to add another chain */
1868 if ((tmp = evbuffer_chain_new(datlen)) == NULL)
1871 if (buf->last_with_datap == &buf->first)
1872 buf->last_with_datap = &tmp->next;
1877 EVUTIL_ASSERT(datlen <= tmp->buffer_len);
1878 tmp->misalign = tmp->buffer_len - datlen;
1880 memcpy(tmp->buffer + tmp->misalign, data, datlen);
1881 buf->total_len += datlen;
1882 buf->n_add_for_cb += (size_t)chain->misalign;
1885 evbuffer_invoke_callbacks_(buf);
1888 EVBUFFER_UNLOCK(buf);
1892 /** Helper: realigns the memory in chain->buffer so that misalign is 0. */
1894 evbuffer_chain_align(struct evbuffer_chain *chain)
1896 EVUTIL_ASSERT(!(chain->flags & EVBUFFER_IMMUTABLE));
1897 EVUTIL_ASSERT(!(chain->flags & EVBUFFER_MEM_PINNED_ANY));
1898 memmove(chain->buffer, chain->buffer + chain->misalign, chain->off);
1899 chain->misalign = 0;
1902 #define MAX_TO_COPY_IN_EXPAND 4096
1903 #define MAX_TO_REALIGN_IN_EXPAND 2048
1905 /** Helper: return true iff we should realign chain to fit datalen bytes of
1908 evbuffer_chain_should_realign(struct evbuffer_chain *chain,
1911 return chain->buffer_len - chain->off >= datlen &&
1912 (chain->off < chain->buffer_len / 2) &&
1913 (chain->off <= MAX_TO_REALIGN_IN_EXPAND);
1916 /* Expands the available space in the event buffer to at least datlen, all in
1917 * a single chunk. Return that chunk. */
1918 static struct evbuffer_chain *
1919 evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen)
1921 struct evbuffer_chain *chain, **chainp;
1922 struct evbuffer_chain *result = NULL;
1923 ASSERT_EVBUFFER_LOCKED(buf);
1925 chainp = buf->last_with_datap;
1927 /* XXX If *chainp is no longer writeable, but has enough space in its
1928 * misalign, this might be a bad idea: we could still use *chainp, not
1929 * (*chainp)->next. */
1930 if (*chainp && CHAIN_SPACE_LEN(*chainp) == 0)
1931 chainp = &(*chainp)->next;
1933 /* 'chain' now points to the first chain with writable space (if any)
1934 * We will either use it, realign it, replace it, or resize it. */
1937 if (chain == NULL ||
1938 (chain->flags & (EVBUFFER_IMMUTABLE|EVBUFFER_MEM_PINNED_ANY))) {
1939 /* We can't use the last_with_data chain at all. Just add a
1940 * new one that's big enough. */
1944 /* If we can fit all the data, then we don't have to do anything */
1945 if (CHAIN_SPACE_LEN(chain) >= datlen) {
1950 /* If the chain is completely empty, just replace it by adding a new
1952 if (chain->off == 0) {
1956 /* If the misalignment plus the remaining space fulfills our data
1957 * needs, we could just force an alignment to happen. Afterwards, we
1958 * have enough space. But only do this if we're saving a lot of space
1959 * and not moving too much data. Otherwise the space savings are
1960 * probably offset by the time lost in copying.
1962 if (evbuffer_chain_should_realign(chain, datlen)) {
1963 evbuffer_chain_align(chain);
1968 /* At this point, we can either resize the last chunk with space in
1969 * it, use the next chunk after it, or If we add a new chunk, we waste
1970 * CHAIN_SPACE_LEN(chain) bytes in the former last chunk. If we
1971 * resize, we have to copy chain->off bytes.
1974 /* Would expanding this chunk be affordable and worthwhile? */
1975 if (CHAIN_SPACE_LEN(chain) < chain->buffer_len / 8 ||
1976 chain->off > MAX_TO_COPY_IN_EXPAND ||
1977 (datlen < EVBUFFER_CHAIN_MAX &&
1978 EVBUFFER_CHAIN_MAX - datlen >= chain->off)) {
1979 /* It's not worth resizing this chain. Can the next one be
1981 if (chain->next && CHAIN_SPACE_LEN(chain->next) >= datlen) {
1982 /* Yes, we can just use the next chain (which should
1984 result = chain->next;
1987 /* No; append a new chain (which will free all
1988 * terminal empty chains.) */
1992 /* Okay, we're going to try to resize this chain: Not doing so
1993 * would waste at least 1/8 of its current allocation, and we
1994 * can do so without having to copy more than
1995 * MAX_TO_COPY_IN_EXPAND bytes. */
1996 /* figure out how much space we need */
1997 size_t length = chain->off + datlen;
1998 struct evbuffer_chain *tmp = evbuffer_chain_new(length);
2002 /* copy the data over that we had so far */
2003 tmp->off = chain->off;
2004 memcpy(tmp->buffer, chain->buffer + chain->misalign,
2006 /* fix up the list */
2007 EVUTIL_ASSERT(*chainp == chain);
2008 result = *chainp = tmp;
2010 if (buf->last == chain)
2013 tmp->next = chain->next;
2014 evbuffer_chain_free(chain);
2019 result = evbuffer_chain_insert_new(buf, datlen);
2023 EVUTIL_ASSERT(result);
2024 EVUTIL_ASSERT(CHAIN_SPACE_LEN(result) >= datlen);
2029 /* Make sure that datlen bytes are available for writing in the last n
2030 * chains. Never copies or moves data. */
2032 evbuffer_expand_fast_(struct evbuffer *buf, size_t datlen, int n)
2034 struct evbuffer_chain *chain = buf->last, *tmp, *next;
2038 ASSERT_EVBUFFER_LOCKED(buf);
2039 EVUTIL_ASSERT(n >= 2);
2041 if (chain == NULL || (chain->flags & EVBUFFER_IMMUTABLE)) {
2042 /* There is no last chunk, or we can't touch the last chunk.
2043 * Just add a new chunk. */
2044 chain = evbuffer_chain_new(datlen);
2048 evbuffer_chain_insert(buf, chain);
2052 used = 0; /* number of chains we're using space in. */
2053 avail = 0; /* how much space they have. */
2054 /* How many bytes can we stick at the end of buffer as it is? Iterate
2055 * over the chains at the end of the buffer, tring to see how much
2056 * space we have in the first n. */
2057 for (chain = *buf->last_with_datap; chain; chain = chain->next) {
2059 size_t space = (size_t) CHAIN_SPACE_LEN(chain);
2060 EVUTIL_ASSERT(chain == *buf->last_with_datap);
2066 /* No data in chain; realign it. */
2067 chain->misalign = 0;
2068 avail += chain->buffer_len;
2071 if (avail >= datlen) {
2072 /* There is already enough space. Just return */
2079 /* There wasn't enough space in the first n chains with space in
2080 * them. Either add a new chain with enough space, or replace all
2081 * empty chains with one that has enough space, depending on n. */
2083 /* The loop ran off the end of the chains before it hit n
2084 * chains; we can add another. */
2085 EVUTIL_ASSERT(chain == NULL);
2087 tmp = evbuffer_chain_new(datlen - avail);
2091 buf->last->next = tmp;
2093 /* (we would only set last_with_data if we added the first
2094 * chain. But if the buffer had no chains, we would have
2095 * just allocated a new chain earlier) */
2098 /* Nuke _all_ the empty chains. */
2099 int rmv_all = 0; /* True iff we removed last_with_data. */
2100 chain = *buf->last_with_datap;
2102 EVUTIL_ASSERT(chain == buf->first);
2106 /* can't overflow, since only mutable chains have
2107 * huge misaligns. */
2108 avail = (size_t) CHAIN_SPACE_LEN(chain);
2109 chain = chain->next;
2113 for (; chain; chain = next) {
2115 EVUTIL_ASSERT(chain->off == 0);
2116 evbuffer_chain_free(chain);
2118 EVUTIL_ASSERT(datlen >= avail);
2119 tmp = evbuffer_chain_new(datlen - avail);
2124 buf->last = *buf->last_with_datap;
2125 (*buf->last_with_datap)->next = NULL;
2131 buf->first = buf->last = tmp;
2132 buf->last_with_datap = &buf->first;
2134 (*buf->last_with_datap)->next = tmp;
2142 evbuffer_expand(struct evbuffer *buf, size_t datlen)
2144 struct evbuffer_chain *chain;
2147 chain = evbuffer_expand_singlechain(buf, datlen);
2148 EVBUFFER_UNLOCK(buf);
2149 return chain ? 0 : -1;
2153 * Reads data from a file descriptor into a buffer.
2156 #if defined(EVENT__HAVE_SYS_UIO_H) || defined(_WIN32)
2157 #define USE_IOVEC_IMPL
2160 #ifdef USE_IOVEC_IMPL
2162 #ifdef EVENT__HAVE_SYS_UIO_H
2163 /* number of iovec we use for writev, fragmentation is going to determine
2164 * how much we end up writing */
2166 #define DEFAULT_WRITE_IOVEC 128
2168 #if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC
2169 #define NUM_WRITE_IOVEC UIO_MAXIOV
2170 #elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC
2171 #define NUM_WRITE_IOVEC IOV_MAX
2173 #define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC
2176 #define IOV_TYPE struct iovec
2177 #define IOV_PTR_FIELD iov_base
2178 #define IOV_LEN_FIELD iov_len
2179 #define IOV_LEN_TYPE size_t
2181 #define NUM_WRITE_IOVEC 16
2182 #define IOV_TYPE WSABUF
2183 #define IOV_PTR_FIELD buf
2184 #define IOV_LEN_FIELD len
2185 #define IOV_LEN_TYPE unsigned long
2188 #define NUM_READ_IOVEC 4
2190 #define EVBUFFER_MAX_READ 4096
2192 /** Helper function to figure out which space to use for reading data into
2193 an evbuffer. Internal use only.
2195 @param buf The buffer to read into
2196 @param howmuch How much we want to read.
2197 @param vecs An array of two or more iovecs or WSABUFs.
2198 @param n_vecs_avail The length of vecs
2199 @param chainp A pointer to a variable to hold the first chain we're
2201 @param exact Boolean: if true, we do not provide more than 'howmuch'
2202 space in the vectors, even if more space is available.
2203 @return The number of buffers we're using.
2206 evbuffer_read_setup_vecs_(struct evbuffer *buf, ev_ssize_t howmuch,
2207 struct evbuffer_iovec *vecs, int n_vecs_avail,
2208 struct evbuffer_chain ***chainp, int exact)
2210 struct evbuffer_chain *chain;
2211 struct evbuffer_chain **firstchainp;
2214 ASSERT_EVBUFFER_LOCKED(buf);
2220 /* Let firstchain be the first chain with any space on it */
2221 firstchainp = buf->last_with_datap;
2222 if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
2223 firstchainp = &(*firstchainp)->next;
2226 chain = *firstchainp;
2227 for (i = 0; i < n_vecs_avail && so_far < (size_t)howmuch; ++i) {
2228 size_t avail = (size_t) CHAIN_SPACE_LEN(chain);
2229 if (avail > (howmuch - so_far) && exact)
2230 avail = howmuch - so_far;
2231 vecs[i].iov_base = CHAIN_SPACE_PTR(chain);
2232 vecs[i].iov_len = avail;
2234 chain = chain->next;
2237 *chainp = firstchainp;
2242 get_n_bytes_readable_on_socket(evutil_socket_t fd)
2244 #if defined(FIONREAD) && defined(_WIN32)
2245 unsigned long lng = EVBUFFER_MAX_READ;
2246 if (ioctlsocket(fd, FIONREAD, &lng) < 0)
2248 /* Can overflow, but mostly harmlessly. XXXX */
2250 #elif defined(FIONREAD)
2251 int n = EVBUFFER_MAX_READ;
2252 if (ioctl(fd, FIONREAD, &n) < 0)
2256 return EVBUFFER_MAX_READ;
2260 /* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t
2263 evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch)
2265 struct evbuffer_chain **chainp;
2269 #ifdef USE_IOVEC_IMPL
2270 int nvecs, i, remaining;
2272 struct evbuffer_chain *chain;
2278 if (buf->freeze_end) {
2283 n = get_n_bytes_readable_on_socket(fd);
2284 if (n <= 0 || n > EVBUFFER_MAX_READ)
2285 n = EVBUFFER_MAX_READ;
2286 if (howmuch < 0 || howmuch > n)
2289 #ifdef USE_IOVEC_IMPL
2290 /* Since we can use iovecs, we're willing to use the last
2291 * NUM_READ_IOVEC chains. */
2292 if (evbuffer_expand_fast_(buf, howmuch, NUM_READ_IOVEC) == -1) {
2296 IOV_TYPE vecs[NUM_READ_IOVEC];
2297 #ifdef EVBUFFER_IOVEC_IS_NATIVE_
2298 nvecs = evbuffer_read_setup_vecs_(buf, howmuch, vecs,
2299 NUM_READ_IOVEC, &chainp, 1);
2301 /* We aren't using the native struct iovec. Therefore,
2303 struct evbuffer_iovec ev_vecs[NUM_READ_IOVEC];
2304 nvecs = evbuffer_read_setup_vecs_(buf, howmuch, ev_vecs, 2,
2307 for (i=0; i < nvecs; ++i)
2308 WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]);
2315 if (WSARecv(fd, vecs, nvecs, &bytesRead, &flags, NULL, NULL)) {
2316 /* The read failed. It might be a close,
2317 * or it might be an error. */
2318 if (WSAGetLastError() == WSAECONNABORTED)
2326 n = readv(fd, vecs, nvecs);
2330 #else /*!USE_IOVEC_IMPL*/
2331 /* If we don't have FIONREAD, we might waste some space here */
2332 /* XXX we _will_ waste some space here if there is any space left
2333 * over on buf->last. */
2334 if ((chain = evbuffer_expand_singlechain(buf, howmuch)) == NULL) {
2339 /* We can append new data at this point */
2340 p = chain->buffer + chain->misalign + chain->off;
2343 n = read(fd, p, howmuch);
2345 n = recv(fd, p, howmuch, 0);
2347 #endif /* USE_IOVEC_IMPL */
2358 #ifdef USE_IOVEC_IMPL
2360 for (i=0; i < nvecs; ++i) {
2361 /* can't overflow, since only mutable chains have
2362 * huge misaligns. */
2363 size_t space = (size_t) CHAIN_SPACE_LEN(*chainp);
2364 /* XXXX This is a kludge that can waste space in perverse
2366 if (space > EVBUFFER_CHAIN_MAX)
2367 space = EVBUFFER_CHAIN_MAX;
2368 if ((ev_ssize_t)space < remaining) {
2369 (*chainp)->off += space;
2370 remaining -= (int)space;
2372 (*chainp)->off += remaining;
2373 buf->last_with_datap = chainp;
2376 chainp = &(*chainp)->next;
2380 advance_last_with_data(buf);
2382 buf->total_len += n;
2383 buf->n_add_for_cb += n;
2385 /* Tell someone about changes in this buffer */
2386 evbuffer_invoke_callbacks_(buf);
2389 EVBUFFER_UNLOCK(buf);
2393 #ifdef USE_IOVEC_IMPL
2395 evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd,
2398 IOV_TYPE iov[NUM_WRITE_IOVEC];
2399 struct evbuffer_chain *chain = buffer->first;
2405 ASSERT_EVBUFFER_LOCKED(buffer);
2406 /* XXX make this top out at some maximal data length? if the
2407 * buffer has (say) 1MB in it, split over 128 chains, there's
2408 * no way it all gets written in one go. */
2409 while (chain != NULL && i < NUM_WRITE_IOVEC && howmuch) {
2411 /* we cannot write the file info via writev */
2412 if (chain->flags & EVBUFFER_SENDFILE)
2415 iov[i].IOV_PTR_FIELD = (void *) (chain->buffer + chain->misalign);
2416 if ((size_t)howmuch >= chain->off) {
2417 /* XXXcould be problematic when windows supports mmap*/
2418 iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)chain->off;
2419 howmuch -= chain->off;
2421 /* XXXcould be problematic when windows supports mmap*/
2422 iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)howmuch;
2425 chain = chain->next;
2433 if (WSASend(fd, iov, i, &bytesSent, 0, NULL, NULL))
2439 n = writev(fd, iov, i);
2447 evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t dest_fd,
2450 struct evbuffer_chain *chain = buffer->first;
2451 struct evbuffer_chain_file_segment *info =
2452 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment,
2454 const int source_fd = info->segment->fd;
2455 #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD)
2457 ev_off_t len = chain->off;
2458 #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS)
2460 ev_off_t offset = chain->misalign;
2463 ASSERT_EVBUFFER_LOCKED(buffer);
2465 #if defined(SENDFILE_IS_MACOSX)
2466 res = sendfile(source_fd, dest_fd, chain->misalign, &len, NULL, 0);
2467 if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
2471 #elif defined(SENDFILE_IS_FREEBSD)
2472 res = sendfile(source_fd, dest_fd, chain->misalign, chain->off, NULL, &len, 0);
2473 if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
2477 #elif defined(SENDFILE_IS_LINUX)
2478 /* TODO(niels): implement splice */
2479 res = sendfile(dest_fd, source_fd, &offset, chain->off);
2480 if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
2481 /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
2485 #elif defined(SENDFILE_IS_SOLARIS)
2487 const off_t offset_orig = offset;
2488 res = sendfile(dest_fd, source_fd, &offset, chain->off);
2489 if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
2490 if (offset - offset_orig)
2491 return offset - offset_orig;
2492 /* if this is EAGAIN or EINTR and no bytes were
2493 * written, return 0 */
2503 evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd,
2508 EVBUFFER_LOCK(buffer);
2510 if (buffer->freeze_start) {
2514 if (howmuch < 0 || (size_t)howmuch > buffer->total_len)
2515 howmuch = buffer->total_len;
2519 struct evbuffer_chain *chain = buffer->first;
2520 if (chain != NULL && (chain->flags & EVBUFFER_SENDFILE))
2521 n = evbuffer_write_sendfile(buffer, fd, howmuch);
2524 #ifdef USE_IOVEC_IMPL
2525 n = evbuffer_write_iovec(buffer, fd, howmuch);
2526 #elif defined(_WIN32)
2527 /* XXX(nickm) Don't disable this code until we know if
2528 * the WSARecv code above works. */
2529 void *p = evbuffer_pullup(buffer, howmuch);
2530 EVUTIL_ASSERT(p || !howmuch);
2531 n = send(fd, p, howmuch, 0);
2533 void *p = evbuffer_pullup(buffer, howmuch);
2534 EVUTIL_ASSERT(p || !howmuch);
2535 n = write(fd, p, howmuch);
2543 evbuffer_drain(buffer, n);
2546 EVBUFFER_UNLOCK(buffer);
2551 evbuffer_write(struct evbuffer *buffer, evutil_socket_t fd)
2553 return evbuffer_write_atmost(buffer, fd, -1);
2557 evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len)
2559 unsigned char *search;
2560 struct evbuffer_ptr ptr;
2562 EVBUFFER_LOCK(buffer);
2564 ptr = evbuffer_search(buffer, (const char *)what, len, NULL);
2568 search = evbuffer_pullup(buffer, ptr.pos + len);
2572 EVBUFFER_UNLOCK(buffer);
2576 /* Subract <b>howfar</b> from the position of <b>pos</b> within
2577 * <b>buf</b>. Returns 0 on success, -1 on failure.
2579 * This isn't exposed yet, because of potential inefficiency issues.
2580 * Maybe it should be. */
2582 evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos,
2587 if (howfar > (size_t)pos->pos)
2589 if (pos->internal_.chain && howfar <= pos->internal_.pos_in_chain) {
2590 pos->internal_.pos_in_chain -= howfar;
2594 const size_t newpos = pos->pos - howfar;
2595 /* Here's the inefficient part: it walks over the
2596 * chains until we hit newpos. */
2597 return evbuffer_ptr_set(buf, pos, newpos, EVBUFFER_PTR_SET);
2602 evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos,
2603 size_t position, enum evbuffer_ptr_how how)
2605 size_t left = position;
2606 struct evbuffer_chain *chain = NULL;
2612 case EVBUFFER_PTR_SET:
2614 pos->pos = position;
2617 case EVBUFFER_PTR_ADD:
2618 /* this avoids iterating over all previous chains if
2619 we just want to advance the position */
2620 if (pos->pos < 0 || EV_SIZE_MAX - position < (size_t)pos->pos) {
2621 EVBUFFER_UNLOCK(buf);
2624 chain = pos->internal_.chain;
2625 pos->pos += position;
2626 position = pos->internal_.pos_in_chain;
2630 EVUTIL_ASSERT(EV_SIZE_MAX - left >= position);
2631 while (chain && position + left >= chain->off) {
2632 left -= chain->off - position;
2633 chain = chain->next;
2637 pos->internal_.chain = chain;
2638 pos->internal_.pos_in_chain = position + left;
2639 } else if (left == 0) {
2640 /* The first byte in the (nonexistent) chain after the last chain */
2641 pos->internal_.chain = NULL;
2642 pos->internal_.pos_in_chain = 0;
2648 EVBUFFER_UNLOCK(buf);
2654 Compare the bytes in buf at position pos to the len bytes in mem. Return
2655 less than 0, 0, or greater than 0 as memcmp.
2658 evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos,
2659 const char *mem, size_t len)
2661 struct evbuffer_chain *chain;
2665 ASSERT_EVBUFFER_LOCKED(buf);
2668 EV_SIZE_MAX - len < (size_t)pos->pos ||
2669 pos->pos + len > buf->total_len)
2672 chain = pos->internal_.chain;
2673 position = pos->internal_.pos_in_chain;
2674 while (len && chain) {
2675 size_t n_comparable;
2676 if (len + position > chain->off)
2677 n_comparable = chain->off - position;
2680 r = memcmp(chain->buffer + chain->misalign + position, mem,
2684 mem += n_comparable;
2685 len -= n_comparable;
2687 chain = chain->next;
2694 evbuffer_search(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start)
2696 return evbuffer_search_range(buffer, what, len, start, NULL);
2700 evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start, const struct evbuffer_ptr *end)
2702 struct evbuffer_ptr pos;
2703 struct evbuffer_chain *chain, *last_chain = NULL;
2704 const unsigned char *p;
2707 EVBUFFER_LOCK(buffer);
2710 memcpy(&pos, start, sizeof(pos));
2711 chain = pos.internal_.chain;
2714 chain = pos.internal_.chain = buffer->first;
2715 pos.internal_.pos_in_chain = 0;
2719 last_chain = end->internal_.chain;
2721 if (!len || len > EV_SSIZE_MAX)
2727 const unsigned char *start_at =
2728 chain->buffer + chain->misalign +
2729 pos.internal_.pos_in_chain;
2730 p = memchr(start_at, first,
2731 chain->off - pos.internal_.pos_in_chain);
2733 pos.pos += p - start_at;
2734 pos.internal_.pos_in_chain += p - start_at;
2735 if (!evbuffer_ptr_memcmp(buffer, &pos, what, len)) {
2736 if (end && pos.pos + (ev_ssize_t)len > end->pos)
2742 ++pos.internal_.pos_in_chain;
2743 if (pos.internal_.pos_in_chain == chain->off) {
2744 chain = pos.internal_.chain = chain->next;
2745 pos.internal_.pos_in_chain = 0;
2748 if (chain == last_chain)
2750 pos.pos += chain->off - pos.internal_.pos_in_chain;
2751 chain = pos.internal_.chain = chain->next;
2752 pos.internal_.pos_in_chain = 0;
2757 PTR_NOT_FOUND(&pos);
2759 EVBUFFER_UNLOCK(buffer);
2764 evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len,
2765 struct evbuffer_ptr *start_at,
2766 struct evbuffer_iovec *vec, int n_vec)
2768 struct evbuffer_chain *chain;
2770 ev_ssize_t len_so_far = 0;
2772 /* Avoid locking in trivial edge cases */
2773 if (start_at && start_at->internal_.chain == NULL)
2776 EVBUFFER_LOCK(buffer);
2779 chain = start_at->internal_.chain;
2780 len_so_far = chain->off
2781 - start_at->internal_.pos_in_chain;
2784 vec[0].iov_base = chain->buffer + chain->misalign
2785 + start_at->internal_.pos_in_chain;
2786 vec[0].iov_len = len_so_far;
2788 chain = chain->next;
2790 chain = buffer->first;
2793 if (n_vec == 0 && len < 0) {
2794 /* If no vectors are provided and they asked for "everything",
2795 * pretend they asked for the actual available amount. */
2796 len = buffer->total_len;
2798 len -= start_at->pos;
2803 if (len >= 0 && len_so_far >= len)
2806 vec[idx].iov_base = chain->buffer + chain->misalign;
2807 vec[idx].iov_len = chain->off;
2812 len_so_far += chain->off;
2813 chain = chain->next;
2816 EVBUFFER_UNLOCK(buffer);
2823 evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap)
2827 int sz, result = -1;
2829 struct evbuffer_chain *chain;
2834 if (buf->freeze_end) {
2838 /* make sure that at least some space is available */
2839 if ((chain = evbuffer_expand_singlechain(buf, 64)) == NULL)
2844 size_t used = chain->misalign + chain->off;
2845 buffer = (char *)chain->buffer + chain->misalign + chain->off;
2846 EVUTIL_ASSERT(chain->buffer_len >= used);
2847 space = chain->buffer_len - used;
2849 buffer = (char*) CHAIN_SPACE_PTR(chain);
2850 space = (size_t) CHAIN_SPACE_LEN(chain);
2853 #define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list))
2857 sz = evutil_vsnprintf(buffer, space, fmt, aq);
2863 if (INT_MAX >= EVBUFFER_CHAIN_MAX &&
2864 (size_t)sz >= EVBUFFER_CHAIN_MAX)
2866 if ((size_t)sz < space) {
2868 buf->total_len += sz;
2869 buf->n_add_for_cb += sz;
2871 advance_last_with_data(buf);
2872 evbuffer_invoke_callbacks_(buf);
2876 if ((chain = evbuffer_expand_singlechain(buf, sz + 1)) == NULL)
2882 EVBUFFER_UNLOCK(buf);
2887 evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...)
2893 res = evbuffer_add_vprintf(buf, fmt, ap);
2900 evbuffer_add_reference(struct evbuffer *outbuf,
2901 const void *data, size_t datlen,
2902 evbuffer_ref_cleanup_cb cleanupfn, void *extra)
2904 struct evbuffer_chain *chain;
2905 struct evbuffer_chain_reference *info;
2908 chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_reference));
2911 chain->flags |= EVBUFFER_REFERENCE | EVBUFFER_IMMUTABLE;
2912 chain->buffer = (u_char *)data;
2913 chain->buffer_len = datlen;
2914 chain->off = datlen;
2916 info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference, chain);
2917 info->cleanupfn = cleanupfn;
2918 info->extra = extra;
2920 EVBUFFER_LOCK(outbuf);
2921 if (outbuf->freeze_end) {
2922 /* don't call chain_free; we do not want to actually invoke
2923 * the cleanup function */
2927 evbuffer_chain_insert(outbuf, chain);
2928 outbuf->n_add_for_cb += datlen;
2930 evbuffer_invoke_callbacks_(outbuf);
2934 EVBUFFER_UNLOCK(outbuf);
2939 /* TODO(niels): we may want to add to automagically convert to mmap, in
2940 * case evbuffer_remove() or evbuffer_pullup() are being used.
2942 struct evbuffer_file_segment *
2943 evbuffer_file_segment_new(
2944 int fd, ev_off_t offset, ev_off_t length, unsigned flags)
2946 struct evbuffer_file_segment *seg =
2947 mm_calloc(sizeof(struct evbuffer_file_segment), 1);
2953 seg->file_offset = offset;
2954 seg->cleanup_cb = NULL;
2955 seg->cleanup_cb_arg = NULL;
2958 #define lseek _lseeki64
2961 #define fstat _fstat
2969 if (fstat(fd, &st) < 0)
2971 length = st.st_size;
2973 seg->length = length;
2975 if (offset < 0 || length < 0 ||
2976 ((ev_uint64_t)length > EVBUFFER_CHAIN_MAX) ||
2977 (ev_uint64_t)offset > (ev_uint64_t)(EVBUFFER_CHAIN_MAX - length))
2980 #if defined(USE_SENDFILE)
2981 if (!(flags & EVBUF_FS_DISABLE_SENDFILE)) {
2982 seg->can_sendfile = 1;
2987 if (evbuffer_file_segment_materialize(seg)<0)
2990 #if defined(USE_SENDFILE)
2993 if (!(flags & EVBUF_FS_DISABLE_LOCKING)) {
2994 EVTHREAD_ALLOC_LOCK(seg->lock, 0);
3002 #ifdef EVENT__HAVE_MMAP
3007 return sysconf(SC_PAGE_SIZE);
3008 #elif defined(_SC_PAGE_SIZE)
3009 return sysconf(_SC_PAGE_SIZE);
3019 evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg)
3021 const unsigned flags = seg->flags;
3022 const int fd = seg->fd;
3023 const ev_off_t length = seg->length;
3024 const ev_off_t offset = seg->file_offset;
3027 return 0; /* already materialized */
3029 #if defined(EVENT__HAVE_MMAP)
3030 if (!(flags & EVBUF_FS_DISABLE_MMAP)) {
3031 off_t offset_rounded = 0, offset_leftover = 0;
3034 /* mmap implementations don't generally like us
3035 * to have an offset that isn't a round */
3036 long page_size = get_page_size();
3037 if (page_size == -1)
3039 offset_leftover = offset % page_size;
3040 offset_rounded = offset - offset_leftover;
3042 mapped = mmap(NULL, length + offset_leftover,
3045 MAP_NOCACHE | /* ??? */
3051 fd, offset_rounded);
3052 if (mapped == MAP_FAILED) {
3053 event_warn("%s: mmap(%d, %d, %zu) failed",
3054 __func__, fd, 0, (size_t)(offset + length));
3056 seg->mapping = mapped;
3057 seg->contents = (char*)mapped+offset_leftover;
3058 seg->mmap_offset = 0;
3059 seg->is_mapping = 1;
3065 if (!(flags & EVBUF_FS_DISABLE_MMAP)) {
3066 intptr_t h = _get_osfhandle(fd);
3068 ev_uint64_t total_size = length+offset;
3069 if ((HANDLE)h == INVALID_HANDLE_VALUE)
3071 m = CreateFileMapping((HANDLE)h, NULL, PAGE_READONLY,
3072 (total_size >> 32), total_size & 0xfffffffful,
3074 if (m != INVALID_HANDLE_VALUE) { /* Does h leak? */
3075 seg->mapping_handle = m;
3076 seg->mmap_offset = offset;
3077 seg->is_mapping = 1;
3083 ev_off_t start_pos = lseek(fd, 0, SEEK_CUR), pos;
3084 ev_off_t read_so_far = 0;
3088 if (!(mem = mm_malloc(length)))
3090 if (start_pos < 0) {
3094 if (lseek(fd, offset, SEEK_SET) < 0) {
3098 while (read_so_far < length) {
3099 n = read(fd, mem+read_so_far, length-read_so_far);
3106 pos = lseek(fd, start_pos, SEEK_SET);
3107 if (n < 0 || (n == 0 && length > read_so_far)) {
3111 } else if (pos < 0) {
3116 seg->contents = mem;
3125 void evbuffer_file_segment_add_cleanup_cb(struct evbuffer_file_segment *seg,
3126 evbuffer_file_segment_cleanup_cb cb, void* arg)
3128 EVUTIL_ASSERT(seg->refcnt > 0);
3129 seg->cleanup_cb = cb;
3130 seg->cleanup_cb_arg = arg;
3134 evbuffer_file_segment_free(struct evbuffer_file_segment *seg)
3137 EVLOCK_LOCK(seg->lock, 0);
3138 refcnt = --seg->refcnt;
3139 EVLOCK_UNLOCK(seg->lock, 0);
3142 EVUTIL_ASSERT(refcnt == 0);
3144 if (seg->is_mapping) {
3146 CloseHandle(seg->mapping_handle);
3147 #elif defined (EVENT__HAVE_MMAP)
3148 off_t offset_leftover;
3149 offset_leftover = seg->file_offset % get_page_size();
3150 if (munmap(seg->mapping, seg->length + offset_leftover) == -1)
3151 event_warn("%s: munmap failed", __func__);
3153 } else if (seg->contents) {
3154 mm_free(seg->contents);
3157 if ((seg->flags & EVBUF_FS_CLOSE_ON_FREE) && seg->fd >= 0) {
3161 if (seg->cleanup_cb) {
3162 (*seg->cleanup_cb)((struct evbuffer_file_segment const*)seg,
3163 seg->flags, seg->cleanup_cb_arg);
3164 seg->cleanup_cb = NULL;
3165 seg->cleanup_cb_arg = NULL;
3168 EVTHREAD_FREE_LOCK(seg->lock, 0);
3173 evbuffer_add_file_segment(struct evbuffer *buf,
3174 struct evbuffer_file_segment *seg, ev_off_t offset, ev_off_t length)
3176 struct evbuffer_chain *chain;
3177 struct evbuffer_chain_file_segment *extra;
3178 int can_use_sendfile = 0;
3181 EVLOCK_LOCK(seg->lock, 0);
3182 if (buf->flags & EVBUFFER_FLAG_DRAINS_TO_FD) {
3183 can_use_sendfile = 1;
3185 if (!seg->contents) {
3186 if (evbuffer_file_segment_materialize(seg)<0) {
3187 EVLOCK_UNLOCK(seg->lock, 0);
3188 EVBUFFER_UNLOCK(buf);
3194 EVLOCK_UNLOCK(seg->lock, 0);
3196 if (buf->freeze_end)
3200 if (offset > seg->length)
3202 length = seg->length - offset;
3205 /* Can we actually add this? */
3206 if (offset+length > seg->length)
3209 chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_file_segment));
3212 extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, chain);
3214 chain->flags |= EVBUFFER_IMMUTABLE|EVBUFFER_FILESEGMENT;
3215 if (can_use_sendfile && seg->can_sendfile) {
3216 chain->flags |= EVBUFFER_SENDFILE;
3217 chain->misalign = seg->file_offset + offset;
3218 chain->off = length;
3219 chain->buffer_len = chain->misalign + length;
3220 } else if (seg->is_mapping) {
3222 ev_uint64_t total_offset = seg->mmap_offset+offset;
3223 ev_uint64_t offset_rounded=0, offset_remaining=0;
3227 memset(&si, 0, sizeof(si)); /* cargo cult */
3229 offset_remaining = total_offset % si.dwAllocationGranularity;
3230 offset_rounded = total_offset - offset_remaining;
3232 data = MapViewOfFile(
3233 seg->mapping_handle,
3235 offset_rounded >> 32,
3236 offset_rounded & 0xfffffffful,
3237 length + offset_remaining);
3242 chain->buffer = (unsigned char*) data;
3243 chain->buffer_len = length+offset_remaining;
3244 chain->misalign = offset_remaining;
3245 chain->off = length;
3247 chain->buffer = (unsigned char*)(seg->contents + offset);
3248 chain->buffer_len = length;
3249 chain->off = length;
3252 chain->buffer = (unsigned char*)(seg->contents + offset);
3253 chain->buffer_len = length;
3254 chain->off = length;
3257 extra->segment = seg;
3258 buf->n_add_for_cb += length;
3259 evbuffer_chain_insert(buf, chain);
3261 evbuffer_invoke_callbacks_(buf);
3263 EVBUFFER_UNLOCK(buf);
3267 EVBUFFER_UNLOCK(buf);
3268 evbuffer_file_segment_free(seg); /* Lowers the refcount */
3273 evbuffer_add_file(struct evbuffer *buf, int fd, ev_off_t offset, ev_off_t length)
3275 struct evbuffer_file_segment *seg;
3276 unsigned flags = EVBUF_FS_CLOSE_ON_FREE;
3279 seg = evbuffer_file_segment_new(fd, offset, length, flags);
3282 r = evbuffer_add_file_segment(buf, seg, 0, length);
3284 evbuffer_file_segment_free(seg);
3289 evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg)
3291 EVBUFFER_LOCK(buffer);
3293 if (!LIST_EMPTY(&buffer->callbacks))
3294 evbuffer_remove_all_callbacks(buffer);
3297 struct evbuffer_cb_entry *ent =
3298 evbuffer_add_cb(buffer, NULL, cbarg);
3299 ent->cb.cb_obsolete = cb;
3300 ent->flags |= EVBUFFER_CB_OBSOLETE;
3302 EVBUFFER_UNLOCK(buffer);
3305 struct evbuffer_cb_entry *
3306 evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
3308 struct evbuffer_cb_entry *e;
3309 if (! (e = mm_calloc(1, sizeof(struct evbuffer_cb_entry))))
3311 EVBUFFER_LOCK(buffer);
3314 e->flags = EVBUFFER_CB_ENABLED;
3315 LIST_INSERT_HEAD(&buffer->callbacks, e, next);
3316 EVBUFFER_UNLOCK(buffer);
3321 evbuffer_remove_cb_entry(struct evbuffer *buffer,
3322 struct evbuffer_cb_entry *ent)
3324 EVBUFFER_LOCK(buffer);
3325 LIST_REMOVE(ent, next);
3326 EVBUFFER_UNLOCK(buffer);
3332 evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
3334 struct evbuffer_cb_entry *cbent;
3336 EVBUFFER_LOCK(buffer);
3337 LIST_FOREACH(cbent, &buffer->callbacks, next) {
3338 if (cb == cbent->cb.cb_func && cbarg == cbent->cbarg) {
3339 result = evbuffer_remove_cb_entry(buffer, cbent);
3344 EVBUFFER_UNLOCK(buffer);
3349 evbuffer_cb_set_flags(struct evbuffer *buffer,
3350 struct evbuffer_cb_entry *cb, ev_uint32_t flags)
3352 /* the user isn't allowed to mess with these. */
3353 flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
3354 EVBUFFER_LOCK(buffer);
3356 EVBUFFER_UNLOCK(buffer);
3361 evbuffer_cb_clear_flags(struct evbuffer *buffer,
3362 struct evbuffer_cb_entry *cb, ev_uint32_t flags)
3364 /* the user isn't allowed to mess with these. */
3365 flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
3366 EVBUFFER_LOCK(buffer);
3367 cb->flags &= ~flags;
3368 EVBUFFER_UNLOCK(buffer);
3373 evbuffer_freeze(struct evbuffer *buffer, int start)
3375 EVBUFFER_LOCK(buffer);
3377 buffer->freeze_start = 1;
3379 buffer->freeze_end = 1;
3380 EVBUFFER_UNLOCK(buffer);
3385 evbuffer_unfreeze(struct evbuffer *buffer, int start)
3387 EVBUFFER_LOCK(buffer);
3389 buffer->freeze_start = 0;
3391 buffer->freeze_end = 0;
3392 EVBUFFER_UNLOCK(buffer);
3398 evbuffer_cb_suspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
3400 if (!(cb->flags & EVBUFFER_CB_SUSPENDED)) {
3401 cb->size_before_suspend = evbuffer_get_length(buffer);
3402 cb->flags |= EVBUFFER_CB_SUSPENDED;
3407 evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
3409 if ((cb->flags & EVBUFFER_CB_SUSPENDED)) {
3410 unsigned call = (cb->flags & EVBUFFER_CB_CALL_ON_UNSUSPEND);
3411 size_t sz = cb->size_before_suspend;
3412 cb->flags &= ~(EVBUFFER_CB_SUSPENDED|
3413 EVBUFFER_CB_CALL_ON_UNSUSPEND);
3414 cb->size_before_suspend = 0;
3415 if (call && (cb->flags & EVBUFFER_CB_ENABLED)) {
3416 cb->cb(buffer, sz, evbuffer_get_length(buffer), cb->cbarg);
3423 evbuffer_get_callbacks_(struct evbuffer *buffer, struct event_callback **cbs,
3427 EVBUFFER_LOCK(buffer);
3428 if (buffer->deferred_cbs) {
3433 cbs[0] = &buffer->deferred;
3437 EVBUFFER_UNLOCK(buffer);