contrib/libevent/bufferevent_openssl.c

   1 /*
   2  * Copyright (c) 2009-2012 Niels Provos and Nick Mathewson
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  * 3. The name of the author may not be used to endorse or promote products
  13  *    derived from this software without specific prior written permission.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 // Get rid of OSX 10.7 and greater deprecation warnings.
  28 #if defined(__APPLE__) && defined(__clang__)
  29 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
  30 #endif
  31
  32 #include "event2/event-config.h"
  33 #include "evconfig-private.h"
  34
  35 #include <sys/types.h>
  36
  37 #ifdef EVENT__HAVE_SYS_TIME_H
  38 #include <sys/time.h>
  39 #endif
  40
  41 #include <errno.h>
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <string.h>
  45 #ifdef EVENT__HAVE_STDARG_H
  46 #include <stdarg.h>
  47 #endif
  48 #ifdef EVENT__HAVE_UNISTD_H
  49 #include <unistd.h>
  50 #endif
  51
  52 #ifdef _WIN32
  53 #include <winsock2.h>
  54 #endif
  55
  56 #include "event2/bufferevent.h"
  57 #include "event2/bufferevent_struct.h"
  58 #include "event2/bufferevent_ssl.h"
  59 #include "event2/buffer.h"
  60 #include "event2/event.h"
  61
  62 #include "mm-internal.h"
  63 #include "bufferevent-internal.h"
  64 #include "log-internal.h"
  65
  66 #include <openssl/bio.h>
  67 #include <openssl/ssl.h>
  68 #include <openssl/err.h>
  69 #include "openssl-compat.h"
  70
  71 /*
  72  * Define an OpenSSL bio that targets a bufferevent.
  73  */
  74
  75 /* --------------------
  76    A BIO is an OpenSSL abstraction that handles reading and writing data.  The
  77    library will happily speak SSL over anything that implements a BIO
  78    interface.
  79
  80    Here we define a BIO implementation that directs its output to a
  81    bufferevent.  We'll want to use this only when none of OpenSSL's built-in
  82    IO mechanisms work for us.
  83    -------------------- */
  84
  85 /* every BIO type needs its own integer type value. */
  86 #define BIO_TYPE_LIBEVENT 57
  87 /* ???? Arguably, we should set BIO_TYPE_FILTER or BIO_TYPE_SOURCE_SINK on
  88  * this. */
  89
  90 #if 0
  91 static void
  92 print_err(int val)
  93 {
  94         int err;
  95         printf("Error was %d\n", val);
  96
  97         while ((err = ERR_get_error())) {
  98                 const char *msg = (const char*)ERR_reason_error_string(err);
  99                 const char *lib = (const char*)ERR_lib_error_string(err);
 100                 const char *func = (const char*)ERR_func_error_string(err);
 101
 102                 printf("%s in %s %s\n", msg, lib, func);
 103         }
 104 }
 105 #else
 106 #define print_err(v) ((void)0)
 107 #endif
 108
 109 /* Called to initialize a new BIO */
 110 static int
 111 bio_bufferevent_new(BIO *b)
 112 {
 113         BIO_set_init(b, 0);
 114         BIO_set_data(b, NULL); /* We'll be putting the bufferevent in this field.*/
 115         return 1;
 116 }
 117
 118 /* Called to uninitialize the BIO. */
 119 static int
 120 bio_bufferevent_free(BIO *b)
 121 {
 122         if (!b)
 123                 return 0;
 124         if (BIO_get_shutdown(b)) {
 125                 if (BIO_get_init(b) && BIO_get_data(b))
 126                         bufferevent_free(BIO_get_data(b));
 127                 BIO_free(b);
 128         }
 129         return 1;
 130 }
 131
 132 /* Called to extract data from the BIO. */
 133 static int
 134 bio_bufferevent_read(BIO *b, char *out, int outlen)
 135 {
 136         int r = 0;
 137         struct evbuffer *input;
 138
 139         BIO_clear_retry_flags(b);
 140
 141         if (!out)
 142                 return 0;
 143         if (!BIO_get_data(b))
 144                 return -1;
 145
 146         input = bufferevent_get_input(BIO_get_data(b));
 147         if (evbuffer_get_length(input) == 0) {
 148                 /* If there's no data to read, say so. */
 149                 BIO_set_retry_read(b);
 150                 return -1;
 151         } else {
 152                 r = evbuffer_remove(input, out, outlen);
 153         }
 154
 155         return r;
 156 }
 157
 158 /* Called to write data info the BIO */
 159 static int
 160 bio_bufferevent_write(BIO *b, const char *in, int inlen)
 161 {
 162         struct bufferevent *bufev = BIO_get_data(b);
 163         struct evbuffer *output;
 164         size_t outlen;
 165
 166         BIO_clear_retry_flags(b);
 167
 168         if (!BIO_get_data(b))
 169                 return -1;
 170
 171         output = bufferevent_get_output(bufev);
 172         outlen = evbuffer_get_length(output);
 173
 174         /* Copy only as much data onto the output buffer as can fit under the
 175          * high-water mark. */
 176         if (bufev->wm_write.high && bufev->wm_write.high <= (outlen+inlen)) {
 177                 if (bufev->wm_write.high <= outlen) {
 178                         /* If no data can fit, we'll need to retry later. */
 179                         BIO_set_retry_write(b);
 180                         return -1;
 181                 }
 182                 inlen = bufev->wm_write.high - outlen;
 183         }
 184
 185         EVUTIL_ASSERT(inlen > 0);
 186         evbuffer_add(output, in, inlen);
 187         return inlen;
 188 }
 189
 190 /* Called to handle various requests */
 191 static long
 192 bio_bufferevent_ctrl(BIO *b, int cmd, long num, void *ptr)
 193 {
 194         struct bufferevent *bufev = BIO_get_data(b);
 195         long ret = 1;
 196
 197         switch (cmd) {
 198         case BIO_CTRL_GET_CLOSE:
 199                 ret = BIO_get_shutdown(b);
 200                 break;
 201         case BIO_CTRL_SET_CLOSE:
 202                 BIO_set_shutdown(b, (int)num);
 203                 break;
 204         case BIO_CTRL_PENDING:
 205                 ret = evbuffer_get_length(bufferevent_get_input(bufev)) != 0;
 206                 break;
 207         case BIO_CTRL_WPENDING:
 208                 ret = evbuffer_get_length(bufferevent_get_output(bufev)) != 0;
 209                 break;
 210         /* XXXX These two are given a special-case treatment because
 211          * of cargo-cultism.  I should come up with a better reason. */
 212         case BIO_CTRL_DUP:
 213         case BIO_CTRL_FLUSH:
 214                 ret = 1;
 215                 break;
 216         default:
 217                 ret = 0;
 218                 break;
 219         }
 220         return ret;
 221 }
 222
 223 /* Called to write a string to the BIO */
 224 static int
 225 bio_bufferevent_puts(BIO *b, const char *s)
 226 {
 227         return bio_bufferevent_write(b, s, strlen(s));
 228 }
 229
 230 /* Method table for the bufferevent BIO */
 231 static BIO_METHOD *methods_bufferevent;
 232
 233 /* Return the method table for the bufferevents BIO */
 234 static BIO_METHOD *
 235 BIO_s_bufferevent(void)
 236 {
 237         if (methods_bufferevent == NULL) {
 238                 methods_bufferevent = BIO_meth_new(BIO_TYPE_LIBEVENT, "bufferevent");
 239                 if (methods_bufferevent == NULL)
 240                         return NULL;
 241                 BIO_meth_set_write(methods_bufferevent, bio_bufferevent_write);
 242                 BIO_meth_set_read(methods_bufferevent, bio_bufferevent_read);
 243                 BIO_meth_set_puts(methods_bufferevent, bio_bufferevent_puts);
 244                 BIO_meth_set_ctrl(methods_bufferevent, bio_bufferevent_ctrl);
 245                 BIO_meth_set_create(methods_bufferevent, bio_bufferevent_new);
 246                 BIO_meth_set_destroy(methods_bufferevent, bio_bufferevent_free);
 247         }
 248         return methods_bufferevent;
 249 }
 250
 251 /* Create a new BIO to wrap communication around a bufferevent.  If close_flag
 252  * is true, the bufferevent will be freed when the BIO is closed. */
 253 static BIO *
 254 BIO_new_bufferevent(struct bufferevent *bufferevent, int close_flag)
 255 {
 256         BIO *result;
 257         if (!bufferevent)
 258                 return NULL;
 259         if (!(result = BIO_new(BIO_s_bufferevent())))
 260                 return NULL;
 261         BIO_set_init(result, 1);
 262         BIO_set_data(result, bufferevent);
 263         BIO_set_shutdown(result, close_flag ? 1 : 0);
 264         return result;
 265 }
 266
 267 /* --------------------
 268    Now, here's the OpenSSL-based implementation of bufferevent.
 269
 270    The implementation comes in two flavors: one that connects its SSL object
 271    to an underlying bufferevent using a BIO_bufferevent, and one that has the
 272    SSL object connect to a socket directly.  The latter should generally be
 273    faster, except on Windows, where your best bet is using a
 274    bufferevent_async.
 275
 276    (OpenSSL supports many other BIO types, too.  But we can't use any unless
 277    we have a good way to get notified when they become readable/writable.)
 278    -------------------- */
 279
 280 struct bio_data_counts {
 281         unsigned long n_written;
 282         unsigned long n_read;
 283 };
 284
 285 struct bufferevent_openssl {
 286         /* Shared fields with common bufferevent implementation code.
 287            If we were set up with an underlying bufferevent, we use the
 288            events here as timers only.  If we have an SSL, then we use
 289            the events as socket events.
 290          */
 291         struct bufferevent_private bev;
 292         /* An underlying bufferevent that we're directing our output to.
 293            If it's NULL, then we're connected to an fd, not an evbuffer. */
 294         struct bufferevent *underlying;
 295         /* The SSL object doing our encryption. */
 296         SSL *ssl;
 297
 298         /* A callback that's invoked when data arrives on our outbuf so we
 299            know to write data to the SSL. */
 300         struct evbuffer_cb_entry *outbuf_cb;
 301
 302         /* A count of how much data the bios have read/written total.  Used
 303            for rate-limiting. */
 304         struct bio_data_counts counts;
 305
 306         /* If this value is greater than 0, then the last SSL_write blocked,
 307          * and we need to try it again with this many bytes. */
 308         ev_ssize_t last_write;
 309
 310 #define NUM_ERRORS 3
 311         ev_uint32_t errors[NUM_ERRORS];
 312
 313         /* When we next get available space, we should say "read" instead of
 314            "write". This can happen if there's a renegotiation during a read
 315            operation. */
 316         unsigned read_blocked_on_write : 1;
 317         /* When we next get data, we should say "write" instead of "read". */
 318         unsigned write_blocked_on_read : 1;
 319         /* Treat TCP close before SSL close on SSL >= v3 as clean EOF. */
 320         unsigned allow_dirty_shutdown : 1;
 321         /* XXX */
 322         unsigned n_errors : 2;
 323
 324         /* Are we currently connecting, accepting, or doing IO? */
 325         unsigned state : 2;
 326         /* If we reset fd, we sould reset state too */
 327         unsigned old_state : 2;
 328 };
 329
 330 static int be_openssl_enable(struct bufferevent *, short);
 331 static int be_openssl_disable(struct bufferevent *, short);
 332 static void be_openssl_unlink(struct bufferevent *);
 333 static void be_openssl_destruct(struct bufferevent *);
 334 static int be_openssl_adj_timeouts(struct bufferevent *);
 335 static int be_openssl_flush(struct bufferevent *bufev,
 336     short iotype, enum bufferevent_flush_mode mode);
 337 static int be_openssl_ctrl(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *);
 338
 339 const struct bufferevent_ops bufferevent_ops_openssl = {
 340         "ssl",
 341         evutil_offsetof(struct bufferevent_openssl, bev.bev),
 342         be_openssl_enable,
 343         be_openssl_disable,
 344         be_openssl_unlink,
 345         be_openssl_destruct,
 346         be_openssl_adj_timeouts,
 347         be_openssl_flush,
 348         be_openssl_ctrl,
 349 };
 350
 351 /* Given a bufferevent, return a pointer to the bufferevent_openssl that
 352  * contains it, if any. */
 353 static inline struct bufferevent_openssl *
 354 upcast(struct bufferevent *bev)
 355 {
 356         struct bufferevent_openssl *bev_o;
 357         if (bev->be_ops != &bufferevent_ops_openssl)
 358                 return NULL;
 359         bev_o = (void*)( ((char*)bev) -
 360                          evutil_offsetof(struct bufferevent_openssl, bev.bev));
 361         EVUTIL_ASSERT(bev_o->bev.bev.be_ops == &bufferevent_ops_openssl);
 362         return bev_o;
 363 }
 364
 365 static inline void
 366 put_error(struct bufferevent_openssl *bev_ssl, unsigned long err)
 367 {
 368         if (bev_ssl->n_errors == NUM_ERRORS)
 369                 return;
 370         /* The error type according to openssl is "unsigned long", but
 371            openssl never uses more than 32 bits of it.  It _can't_ use more
 372            than 32 bits of it, since it needs to report errors on systems
 373            where long is only 32 bits.
 374          */
 375         bev_ssl->errors[bev_ssl->n_errors++] = (ev_uint32_t) err;
 376 }
 377
 378 /* Have the base communications channel (either the underlying bufferevent or
 379  * ev_read and ev_write) start reading.  Take the read-blocked-on-write flag
 380  * into account. */
 381 static int
 382 start_reading(struct bufferevent_openssl *bev_ssl)
 383 {
 384         if (bev_ssl->underlying) {
 385                 bufferevent_unsuspend_read_(bev_ssl->underlying,
 386                     BEV_SUSPEND_FILT_READ);
 387                 return 0;
 388         } else {
 389                 struct bufferevent *bev = &bev_ssl->bev.bev;
 390                 int r;
 391                 r = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read);
 392                 if (r == 0 && bev_ssl->read_blocked_on_write)
 393                         r = bufferevent_add_event_(&bev->ev_write,
 394                             &bev->timeout_write);
 395                 return r;
 396         }
 397 }
 398
 399 /* Have the base communications channel (either the underlying bufferevent or
 400  * ev_read and ev_write) start writing.  Take the write-blocked-on-read flag
 401  * into account. */
 402 static int
 403 start_writing(struct bufferevent_openssl *bev_ssl)
 404 {
 405         int r = 0;
 406         if (bev_ssl->underlying) {
 407                 if (bev_ssl->write_blocked_on_read) {
 408                         bufferevent_unsuspend_read_(bev_ssl->underlying,
 409                             BEV_SUSPEND_FILT_READ);
 410                 }
 411         } else {
 412                 struct bufferevent *bev = &bev_ssl->bev.bev;
 413                 r = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write);
 414                 if (!r && bev_ssl->write_blocked_on_read)
 415                         r = bufferevent_add_event_(&bev->ev_read,
 416                             &bev->timeout_read);
 417         }
 418         return r;
 419 }
 420
 421 static void
 422 stop_reading(struct bufferevent_openssl *bev_ssl)
 423 {
 424         if (bev_ssl->write_blocked_on_read)
 425                 return;
 426         if (bev_ssl->underlying) {
 427                 bufferevent_suspend_read_(bev_ssl->underlying,
 428                     BEV_SUSPEND_FILT_READ);
 429         } else {
 430                 struct bufferevent *bev = &bev_ssl->bev.bev;
 431                 event_del(&bev->ev_read);
 432         }
 433 }
 434
 435 static void
 436 stop_writing(struct bufferevent_openssl *bev_ssl)
 437 {
 438         if (bev_ssl->read_blocked_on_write)
 439                 return;
 440         if (bev_ssl->underlying) {
 441                 bufferevent_unsuspend_read_(bev_ssl->underlying,
 442                     BEV_SUSPEND_FILT_READ);
 443         } else {
 444                 struct bufferevent *bev = &bev_ssl->bev.bev;
 445                 event_del(&bev->ev_write);
 446         }
 447 }
 448
 449 static int
 450 set_rbow(struct bufferevent_openssl *bev_ssl)
 451 {
 452         if (!bev_ssl->underlying)
 453                 stop_reading(bev_ssl);
 454         bev_ssl->read_blocked_on_write = 1;
 455         return start_writing(bev_ssl);
 456 }
 457
 458 static int
 459 set_wbor(struct bufferevent_openssl *bev_ssl)
 460 {
 461         if (!bev_ssl->underlying)
 462                 stop_writing(bev_ssl);
 463         bev_ssl->write_blocked_on_read = 1;
 464         return start_reading(bev_ssl);
 465 }
 466
 467 static int
 468 clear_rbow(struct bufferevent_openssl *bev_ssl)
 469 {
 470         struct bufferevent *bev = &bev_ssl->bev.bev;
 471         int r = 0;
 472         bev_ssl->read_blocked_on_write = 0;
 473         if (!(bev->enabled & EV_WRITE))
 474                 stop_writing(bev_ssl);
 475         if (bev->enabled & EV_READ)
 476                 r = start_reading(bev_ssl);
 477         return r;
 478 }
 479
 480
 481 static int
 482 clear_wbor(struct bufferevent_openssl *bev_ssl)
 483 {
 484         struct bufferevent *bev = &bev_ssl->bev.bev;
 485         int r = 0;
 486         bev_ssl->write_blocked_on_read = 0;
 487         if (!(bev->enabled & EV_READ))
 488                 stop_reading(bev_ssl);
 489         if (bev->enabled & EV_WRITE)
 490                 r = start_writing(bev_ssl);
 491         return r;
 492 }
 493
 494 static void
 495 conn_closed(struct bufferevent_openssl *bev_ssl, int when, int errcode, int ret)
 496 {
 497         int event = BEV_EVENT_ERROR;
 498         int dirty_shutdown = 0;
 499         unsigned long err;
 500
 501         switch (errcode) {
 502         case SSL_ERROR_ZERO_RETURN:
 503                 /* Possibly a clean shutdown. */
 504                 if (SSL_get_shutdown(bev_ssl->ssl) & SSL_RECEIVED_SHUTDOWN)
 505                         event = BEV_EVENT_EOF;
 506                 else
 507                         dirty_shutdown = 1;
 508                 break;
 509         case SSL_ERROR_SYSCALL:
 510                 /* IO error; possibly a dirty shutdown. */
 511                 if ((ret == 0 || ret == -1) && ERR_peek_error() == 0)
 512                         dirty_shutdown = 1;
 513                 break;
 514         case SSL_ERROR_SSL:
 515                 /* Protocol error. */
 516                 break;
 517         case SSL_ERROR_WANT_X509_LOOKUP:
 518                 /* XXXX handle this. */
 519                 break;
 520         case SSL_ERROR_NONE:
 521         case SSL_ERROR_WANT_READ:
 522         case SSL_ERROR_WANT_WRITE:
 523         case SSL_ERROR_WANT_CONNECT:
 524         case SSL_ERROR_WANT_ACCEPT:
 525         default:
 526                 /* should be impossible; treat as normal error. */
 527                 event_warnx("BUG: Unexpected OpenSSL error code %d", errcode);
 528                 break;
 529         }
 530
 531         while ((err = ERR_get_error())) {
 532                 put_error(bev_ssl, err);
 533         }
 534
 535         if (dirty_shutdown && bev_ssl->allow_dirty_shutdown)
 536                 event = BEV_EVENT_EOF;
 537
 538         stop_reading(bev_ssl);
 539         stop_writing(bev_ssl);
 540
 541         /* when is BEV_EVENT_{READING|WRITING} */
 542         event = when | event;
 543         bufferevent_run_eventcb_(&bev_ssl->bev.bev, event, 0);
 544 }
 545
 546 static void
 547 init_bio_counts(struct bufferevent_openssl *bev_ssl)
 548 {
 549         BIO *rbio, *wbio;
 550
 551         wbio = SSL_get_wbio(bev_ssl->ssl);
 552         bev_ssl->counts.n_written = wbio ? BIO_number_written(wbio) : 0;
 553         rbio = SSL_get_rbio(bev_ssl->ssl);
 554         bev_ssl->counts.n_read = rbio ? BIO_number_read(rbio) : 0;
 555 }
 556
 557 static inline void
 558 decrement_buckets(struct bufferevent_openssl *bev_ssl)
 559 {
 560         unsigned long num_w = BIO_number_written(SSL_get_wbio(bev_ssl->ssl));
 561         unsigned long num_r = BIO_number_read(SSL_get_rbio(bev_ssl->ssl));
 562         /* These next two subtractions can wrap around. That's okay. */
 563         unsigned long w = num_w - bev_ssl->counts.n_written;
 564         unsigned long r = num_r - bev_ssl->counts.n_read;
 565         if (w)
 566                 bufferevent_decrement_write_buckets_(&bev_ssl->bev, w);
 567         if (r)
 568                 bufferevent_decrement_read_buckets_(&bev_ssl->bev, r);
 569         bev_ssl->counts.n_written = num_w;
 570         bev_ssl->counts.n_read = num_r;
 571 }
 572
 573 #define OP_MADE_PROGRESS 1
 574 #define OP_BLOCKED 2
 575 #define OP_ERR 4
 576
 577 /* Return a bitmask of OP_MADE_PROGRESS (if we read anything); OP_BLOCKED (if
 578    we're now blocked); and OP_ERR (if an error occurred). */
 579 static int
 580 do_read(struct bufferevent_openssl *bev_ssl, int n_to_read) {
 581         /* Requires lock */
 582         struct bufferevent *bev = &bev_ssl->bev.bev;
 583         struct evbuffer *input = bev->input;
 584         int r, n, i, n_used = 0, atmost;
 585         struct evbuffer_iovec space[2];
 586         int result = 0;
 587
 588         if (bev_ssl->bev.read_suspended)
 589                 return 0;
 590
 591         atmost = bufferevent_get_read_max_(&bev_ssl->bev);
 592         if (n_to_read > atmost)
 593                 n_to_read = atmost;
 594
 595         n = evbuffer_reserve_space(input, n_to_read, space, 2);
 596         if (n < 0)
 597                 return OP_ERR;
 598
 599         for (i=0; i<n; ++i) {
 600                 if (bev_ssl->bev.read_suspended)
 601                         break;
 602                 ERR_clear_error();
 603                 r = SSL_read(bev_ssl->ssl, space[i].iov_base, space[i].iov_len);
 604                 if (r>0) {
 605                         result |= OP_MADE_PROGRESS;
 606                         if (bev_ssl->read_blocked_on_write)
 607                                 if (clear_rbow(bev_ssl) < 0)
 608                                         return OP_ERR | result;
 609                         ++n_used;
 610                         space[i].iov_len = r;
 611                         decrement_buckets(bev_ssl);
 612                 } else {
 613                         int err = SSL_get_error(bev_ssl->ssl, r);
 614                         print_err(err);
 615                         switch (err) {
 616                         case SSL_ERROR_WANT_READ:
 617                                 /* Can't read until underlying has more data. */
 618                                 if (bev_ssl->read_blocked_on_write)
 619                                         if (clear_rbow(bev_ssl) < 0)
 620                                                 return OP_ERR | result;
 621                                 break;
 622                         case SSL_ERROR_WANT_WRITE:
 623                                 /* This read operation requires a write, and the
 624                                  * underlying is full */
 625                                 if (!bev_ssl->read_blocked_on_write)
 626                                         if (set_rbow(bev_ssl) < 0)
 627                                                 return OP_ERR | result;
 628                                 break;
 629                         default:
 630                                 conn_closed(bev_ssl, BEV_EVENT_READING, err, r);
 631                                 break;
 632                         }
 633                         result |= OP_BLOCKED;
 634                         break; /* out of the loop */
 635                 }
 636         }
 637
 638         if (n_used) {
 639                 evbuffer_commit_space(input, space, n_used);
 640                 if (bev_ssl->underlying)
 641                         BEV_RESET_GENERIC_READ_TIMEOUT(bev);
 642         }
 643
 644         return result;
 645 }
 646
 647 /* Return a bitmask of OP_MADE_PROGRESS (if we wrote anything); OP_BLOCKED (if
 648    we're now blocked); and OP_ERR (if an error occurred). */
 649 static int
 650 do_write(struct bufferevent_openssl *bev_ssl, int atmost)
 651 {
 652         int i, r, n, n_written = 0;
 653         struct bufferevent *bev = &bev_ssl->bev.bev;
 654         struct evbuffer *output = bev->output;
 655         struct evbuffer_iovec space[8];
 656         int result = 0;
 657
 658         if (bev_ssl->last_write > 0)
 659                 atmost = bev_ssl->last_write;
 660         else
 661                 atmost = bufferevent_get_write_max_(&bev_ssl->bev);
 662
 663         n = evbuffer_peek(output, atmost, NULL, space, 8);
 664         if (n < 0)
 665                 return OP_ERR | result;
 666
 667         if (n > 8)
 668                 n = 8;
 669         for (i=0; i < n; ++i) {
 670                 if (bev_ssl->bev.write_suspended)
 671                         break;
 672
 673                 /* SSL_write will (reasonably) return 0 if we tell it to
 674                    send 0 data.  Skip this case so we don't interpret the
 675                    result as an error */
 676                 if (space[i].iov_len == 0)
 677                         continue;
 678
 679                 ERR_clear_error();
 680                 r = SSL_write(bev_ssl->ssl, space[i].iov_base,
 681                     space[i].iov_len);
 682                 if (r > 0) {
 683                         result |= OP_MADE_PROGRESS;
 684                         if (bev_ssl->write_blocked_on_read)
 685                                 if (clear_wbor(bev_ssl) < 0)
 686                                         return OP_ERR | result;
 687                         n_written += r;
 688                         bev_ssl->last_write = -1;
 689                         decrement_buckets(bev_ssl);
 690                 } else {
 691                         int err = SSL_get_error(bev_ssl->ssl, r);
 692                         print_err(err);
 693                         switch (err) {
 694                         case SSL_ERROR_WANT_WRITE:
 695                                 /* Can't read until underlying has more data. */
 696                                 if (bev_ssl->write_blocked_on_read)
 697                                         if (clear_wbor(bev_ssl) < 0)
 698                                                 return OP_ERR | result;
 699                                 bev_ssl->last_write = space[i].iov_len;
 700                                 break;
 701                         case SSL_ERROR_WANT_READ:
 702                                 /* This read operation requires a write, and the
 703                                  * underlying is full */
 704                                 if (!bev_ssl->write_blocked_on_read)
 705                                         if (set_wbor(bev_ssl) < 0)
 706                                                 return OP_ERR | result;
 707                                 bev_ssl->last_write = space[i].iov_len;
 708                                 break;
 709                         default:
 710                                 conn_closed(bev_ssl, BEV_EVENT_WRITING, err, r);
 711                                 bev_ssl->last_write = -1;
 712                                 break;
 713                         }
 714                         result |= OP_BLOCKED;
 715                         break;
 716                 }
 717         }
 718         if (n_written) {
 719                 evbuffer_drain(output, n_written);
 720                 if (bev_ssl->underlying)
 721                         BEV_RESET_GENERIC_WRITE_TIMEOUT(bev);
 722
 723                 bufferevent_trigger_nolock_(bev, EV_WRITE, BEV_OPT_DEFER_CALLBACKS);
 724         }
 725         return result;
 726 }
 727
 728 #define WRITE_FRAME 15000
 729
 730 #define READ_DEFAULT 4096
 731
 732 /* Try to figure out how many bytes to read; return 0 if we shouldn't be
 733  * reading. */
 734 static int
 735 bytes_to_read(struct bufferevent_openssl *bev)
 736 {
 737         struct evbuffer *input = bev->bev.bev.input;
 738         struct event_watermark *wm = &bev->bev.bev.wm_read;
 739         int result = READ_DEFAULT;
 740         ev_ssize_t limit;
 741         /* XXX 99% of this is generic code that nearly all bufferevents will
 742          * want. */
 743
 744         if (bev->write_blocked_on_read) {
 745                 return 0;
 746         }
 747
 748         if (! (bev->bev.bev.enabled & EV_READ)) {
 749                 return 0;
 750         }
 751
 752         if (bev->bev.read_suspended) {
 753                 return 0;
 754         }
 755
 756         if (wm->high) {
 757                 if (evbuffer_get_length(input) >= wm->high) {
 758                         return 0;
 759                 }
 760
 761                 result = wm->high - evbuffer_get_length(input);
 762         } else {
 763                 result = READ_DEFAULT;
 764         }
 765
 766         /* Respect the rate limit */
 767         limit = bufferevent_get_read_max_(&bev->bev);
 768         if (result > limit) {
 769                 result = limit;
 770         }
 771
 772         return result;
 773 }
 774
 775
 776 /* Things look readable.  If write is blocked on read, write till it isn't.
 777  * Read from the underlying buffer until we block or we hit our high-water
 778  * mark.
 779  */
 780 static void
 781 consider_reading(struct bufferevent_openssl *bev_ssl)
 782 {
 783         int r;
 784         int n_to_read;
 785         int all_result_flags = 0;
 786
 787         while (bev_ssl->write_blocked_on_read) {
 788                 r = do_write(bev_ssl, WRITE_FRAME);
 789                 if (r & (OP_BLOCKED|OP_ERR))
 790                         break;
 791         }
 792         if (bev_ssl->write_blocked_on_read)
 793                 return;
 794
 795         n_to_read = bytes_to_read(bev_ssl);
 796
 797         while (n_to_read) {
 798                 r = do_read(bev_ssl, n_to_read);
 799                 all_result_flags |= r;
 800
 801                 if (r & (OP_BLOCKED|OP_ERR))
 802                         break;
 803
 804                 if (bev_ssl->bev.read_suspended)
 805                         break;
 806
 807                 /* Read all pending data.  This won't hit the network
 808                  * again, and will (most importantly) put us in a state
 809                  * where we don't need to read anything else until the
 810                  * socket is readable again.  It'll potentially make us
 811                  * overrun our read high-watermark (somewhat
 812                  * regrettable).  The damage to the rate-limit has
 813                  * already been done, since OpenSSL went and read a
 814                  * whole SSL record anyway. */
 815                 n_to_read = SSL_pending(bev_ssl->ssl);
 816
 817                 /* XXX This if statement is actually a bad bug, added to avoid
 818                  * XXX a worse bug.
 819                  *
 820                  * The bad bug: It can potentially cause resource unfairness
 821                  * by reading too much data from the underlying bufferevent;
 822                  * it can potentially cause read looping if the underlying
 823                  * bufferevent is a bufferevent_pair and deferred callbacks
 824                  * aren't used.
 825                  *
 826                  * The worse bug: If we didn't do this, then we would
 827                  * potentially not read any more from bev_ssl->underlying
 828                  * until more data arrived there, which could lead to us
 829                  * waiting forever.
 830                  */
 831                 if (!n_to_read && bev_ssl->underlying)
 832                         n_to_read = bytes_to_read(bev_ssl);
 833         }
 834
 835         if (all_result_flags & OP_MADE_PROGRESS) {
 836                 struct bufferevent *bev = &bev_ssl->bev.bev;
 837
 838                 bufferevent_trigger_nolock_(bev, EV_READ, 0);
 839         }
 840
 841         if (!bev_ssl->underlying) {
 842                 /* Should be redundant, but let's avoid busy-looping */
 843                 if (bev_ssl->bev.read_suspended ||
 844                     !(bev_ssl->bev.bev.enabled & EV_READ)) {
 845                         event_del(&bev_ssl->bev.bev.ev_read);
 846                 }
 847         }
 848 }
 849
 850 static void
 851 consider_writing(struct bufferevent_openssl *bev_ssl)
 852 {
 853         int r;
 854         struct evbuffer *output = bev_ssl->bev.bev.output;
 855         struct evbuffer *target = NULL;
 856         struct event_watermark *wm = NULL;
 857
 858         while (bev_ssl->read_blocked_on_write) {
 859                 r = do_read(bev_ssl, 1024); /* XXXX 1024 is a hack */
 860                 if (r & OP_MADE_PROGRESS) {
 861                         struct bufferevent *bev = &bev_ssl->bev.bev;
 862
 863                         bufferevent_trigger_nolock_(bev, EV_READ, 0);
 864                 }
 865                 if (r & (OP_ERR|OP_BLOCKED))
 866                         break;
 867         }
 868         if (bev_ssl->read_blocked_on_write)
 869                 return;
 870         if (bev_ssl->underlying) {
 871                 target = bev_ssl->underlying->output;
 872                 wm = &bev_ssl->underlying->wm_write;
 873         }
 874         while ((bev_ssl->bev.bev.enabled & EV_WRITE) &&
 875             (! bev_ssl->bev.write_suspended) &&
 876             evbuffer_get_length(output) &&
 877             (!target || (! wm->high || evbuffer_get_length(target) < wm->high))) {
 878                 int n_to_write;
 879                 if (wm && wm->high)
 880                         n_to_write = wm->high - evbuffer_get_length(target);
 881                 else
 882                         n_to_write = WRITE_FRAME;
 883                 r = do_write(bev_ssl, n_to_write);
 884                 if (r & (OP_BLOCKED|OP_ERR))
 885                         break;
 886         }
 887
 888         if (!bev_ssl->underlying) {
 889                 if (evbuffer_get_length(output) == 0) {
 890                         event_del(&bev_ssl->bev.bev.ev_write);
 891                 } else if (bev_ssl->bev.write_suspended ||
 892                     !(bev_ssl->bev.bev.enabled & EV_WRITE)) {
 893                         /* Should be redundant, but let's avoid busy-looping */
 894                         event_del(&bev_ssl->bev.bev.ev_write);
 895                 }
 896         }
 897 }
 898
 899 static void
 900 be_openssl_readcb(struct bufferevent *bev_base, void *ctx)
 901 {
 902         struct bufferevent_openssl *bev_ssl = ctx;
 903         consider_reading(bev_ssl);
 904 }
 905
 906 static void
 907 be_openssl_writecb(struct bufferevent *bev_base, void *ctx)
 908 {
 909         struct bufferevent_openssl *bev_ssl = ctx;
 910         consider_writing(bev_ssl);
 911 }
 912
 913 static void
 914 be_openssl_eventcb(struct bufferevent *bev_base, short what, void *ctx)
 915 {
 916         struct bufferevent_openssl *bev_ssl = ctx;
 917         int event = 0;
 918
 919         if (what & BEV_EVENT_EOF) {
 920                 if (bev_ssl->allow_dirty_shutdown)
 921                         event = BEV_EVENT_EOF;
 922                 else
 923                         event = BEV_EVENT_ERROR;
 924         } else if (what & BEV_EVENT_TIMEOUT) {
 925                 /* We sure didn't set this.  Propagate it to the user. */
 926                 event = what;
 927         } else if (what & BEV_EVENT_ERROR) {
 928                 /* An error occurred on the connection.  Propagate it to the user. */
 929                 event = what;
 930         } else if (what & BEV_EVENT_CONNECTED) {
 931                 /* Ignore it.  We're saying SSL_connect() already, which will
 932                    eat it. */
 933         }
 934         if (event)
 935                 bufferevent_run_eventcb_(&bev_ssl->bev.bev, event, 0);
 936 }
 937
 938 static void
 939 be_openssl_readeventcb(evutil_socket_t fd, short what, void *ptr)
 940 {
 941         struct bufferevent_openssl *bev_ssl = ptr;
 942         bufferevent_incref_and_lock_(&bev_ssl->bev.bev);
 943         if (what == EV_TIMEOUT) {
 944                 bufferevent_run_eventcb_(&bev_ssl->bev.bev,
 945                     BEV_EVENT_TIMEOUT|BEV_EVENT_READING, 0);
 946         } else {
 947                 consider_reading(bev_ssl);
 948         }
 949         bufferevent_decref_and_unlock_(&bev_ssl->bev.bev);
 950 }
 951
 952 static void
 953 be_openssl_writeeventcb(evutil_socket_t fd, short what, void *ptr)
 954 {
 955         struct bufferevent_openssl *bev_ssl = ptr;
 956         bufferevent_incref_and_lock_(&bev_ssl->bev.bev);
 957         if (what == EV_TIMEOUT) {
 958                 bufferevent_run_eventcb_(&bev_ssl->bev.bev,
 959                     BEV_EVENT_TIMEOUT|BEV_EVENT_WRITING, 0);
 960         } else {
 961                 consider_writing(bev_ssl);
 962         }
 963         bufferevent_decref_and_unlock_(&bev_ssl->bev.bev);
 964 }
 965
 966 static int
 967 be_openssl_auto_fd(struct bufferevent_openssl *bev_ssl, int fd)
 968 {
 969         if (!bev_ssl->underlying) {
 970                 struct bufferevent *bev = &bev_ssl->bev.bev;
 971                 if (event_initialized(&bev->ev_read) && fd < 0) {
 972                         fd = event_get_fd(&bev->ev_read);
 973                 }
 974         }
 975         return fd;
 976 }
 977
 978 static int
 979 set_open_callbacks(struct bufferevent_openssl *bev_ssl, evutil_socket_t fd)
 980 {
 981         if (bev_ssl->underlying) {
 982                 bufferevent_setcb(bev_ssl->underlying,
 983                     be_openssl_readcb, be_openssl_writecb, be_openssl_eventcb,
 984                     bev_ssl);
 985                 return 0;
 986         } else {
 987                 struct bufferevent *bev = &bev_ssl->bev.bev;
 988                 int rpending=0, wpending=0, r1=0, r2=0;
 989
 990                 if (event_initialized(&bev->ev_read)) {
 991                         rpending = event_pending(&bev->ev_read, EV_READ, NULL);
 992                         wpending = event_pending(&bev->ev_write, EV_WRITE, NULL);
 993
 994                         event_del(&bev->ev_read);
 995                         event_del(&bev->ev_write);
 996                 }
 997
 998                 event_assign(&bev->ev_read, bev->ev_base, fd,
 999                     EV_READ|EV_PERSIST|EV_FINALIZE,
1000                     be_openssl_readeventcb, bev_ssl);
1001                 event_assign(&bev->ev_write, bev->ev_base, fd,
1002                     EV_WRITE|EV_PERSIST|EV_FINALIZE,
1003                     be_openssl_writeeventcb, bev_ssl);
1004
1005                 if (rpending)
1006                         r1 = bufferevent_add_event_(&bev->ev_read, &bev->timeout_read);
1007                 if (wpending)
1008                         r2 = bufferevent_add_event_(&bev->ev_write, &bev->timeout_write);
1009
1010                 return (r1<0 || r2<0) ? -1 : 0;
1011         }
1012 }
1013
1014 static int
1015 do_handshake(struct bufferevent_openssl *bev_ssl)
1016 {
1017         int r;
1018
1019         switch (bev_ssl->state) {
1020         default:
1021         case BUFFEREVENT_SSL_OPEN:
1022                 EVUTIL_ASSERT(0);
1023                 return -1;
1024         case BUFFEREVENT_SSL_CONNECTING:
1025         case BUFFEREVENT_SSL_ACCEPTING:
1026                 ERR_clear_error();
1027                 r = SSL_do_handshake(bev_ssl->ssl);
1028                 break;
1029         }
1030         decrement_buckets(bev_ssl);
1031
1032         if (r==1) {
1033                 int fd = event_get_fd(&bev_ssl->bev.bev.ev_read);
1034                 /* We're done! */
1035                 bev_ssl->state = BUFFEREVENT_SSL_OPEN;
1036                 set_open_callbacks(bev_ssl, fd); /* XXXX handle failure */
1037                 /* Call do_read and do_write as needed */
1038                 bufferevent_enable(&bev_ssl->bev.bev, bev_ssl->bev.bev.enabled);
1039                 bufferevent_run_eventcb_(&bev_ssl->bev.bev,
1040                     BEV_EVENT_CONNECTED, 0);
1041                 return 1;
1042         } else {
1043                 int err = SSL_get_error(bev_ssl->ssl, r);
1044                 print_err(err);
1045                 switch (err) {
1046                 case SSL_ERROR_WANT_WRITE:
1047                         stop_reading(bev_ssl);
1048                         return start_writing(bev_ssl);
1049                 case SSL_ERROR_WANT_READ:
1050                         stop_writing(bev_ssl);
1051                         return start_reading(bev_ssl);
1052                 default:
1053                         conn_closed(bev_ssl, BEV_EVENT_READING, err, r);
1054                         return -1;
1055                 }
1056         }
1057 }
1058
1059 static void
1060 be_openssl_handshakecb(struct bufferevent *bev_base, void *ctx)
1061 {
1062         struct bufferevent_openssl *bev_ssl = ctx;
1063         do_handshake(bev_ssl);/* XXX handle failure */
1064 }
1065
1066 static void
1067 be_openssl_handshakeeventcb(evutil_socket_t fd, short what, void *ptr)
1068 {
1069         struct bufferevent_openssl *bev_ssl = ptr;
1070
1071         bufferevent_incref_and_lock_(&bev_ssl->bev.bev);
1072         if (what & EV_TIMEOUT) {
1073                 bufferevent_run_eventcb_(&bev_ssl->bev.bev, BEV_EVENT_TIMEOUT, 0);
1074         } else
1075                 do_handshake(bev_ssl);/* XXX handle failure */
1076         bufferevent_decref_and_unlock_(&bev_ssl->bev.bev);
1077 }
1078
1079 static int
1080 set_handshake_callbacks(struct bufferevent_openssl *bev_ssl, evutil_socket_t fd)
1081 {
1082         if (bev_ssl->underlying) {
1083                 bufferevent_setcb(bev_ssl->underlying,
1084                     be_openssl_handshakecb, be_openssl_handshakecb,
1085                     be_openssl_eventcb,
1086                     bev_ssl);
1087
1088                 if (fd < 0)
1089                         return 0;
1090
1091                 if (bufferevent_setfd(bev_ssl->underlying, fd))
1092                         return 1;
1093
1094                 return do_handshake(bev_ssl);
1095         } else {
1096                 struct bufferevent *bev = &bev_ssl->bev.bev;
1097
1098                 if (event_initialized(&bev->ev_read)) {
1099                         event_del(&bev->ev_read);
1100                         event_del(&bev->ev_write);
1101                 }
1102
1103                 event_assign(&bev->ev_read, bev->ev_base, fd,
1104                     EV_READ|EV_PERSIST|EV_FINALIZE,
1105                     be_openssl_handshakeeventcb, bev_ssl);
1106                 event_assign(&bev->ev_write, bev->ev_base, fd,
1107                     EV_WRITE|EV_PERSIST|EV_FINALIZE,
1108                     be_openssl_handshakeeventcb, bev_ssl);
1109                 if (fd >= 0)
1110                         bufferevent_enable(bev, bev->enabled);
1111                 return 0;
1112         }
1113 }
1114
1115 int
1116 bufferevent_ssl_renegotiate(struct bufferevent *bev)
1117 {
1118         struct bufferevent_openssl *bev_ssl = upcast(bev);
1119         if (!bev_ssl)
1120                 return -1;
1121         if (SSL_renegotiate(bev_ssl->ssl) < 0)
1122                 return -1;
1123         bev_ssl->state = BUFFEREVENT_SSL_CONNECTING;
1124         if (set_handshake_callbacks(bev_ssl, be_openssl_auto_fd(bev_ssl, -1)) < 0)
1125                 return -1;
1126         if (!bev_ssl->underlying)
1127                 return do_handshake(bev_ssl);
1128         return 0;
1129 }
1130
1131 static void
1132 be_openssl_outbuf_cb(struct evbuffer *buf,
1133     const struct evbuffer_cb_info *cbinfo, void *arg)
1134 {
1135         struct bufferevent_openssl *bev_ssl = arg;
1136         int r = 0;
1137         /* XXX need to hold a reference here. */
1138
1139         if (cbinfo->n_added && bev_ssl->state == BUFFEREVENT_SSL_OPEN) {
1140                 if (cbinfo->orig_size == 0)
1141                         r = bufferevent_add_event_(&bev_ssl->bev.bev.ev_write,
1142                             &bev_ssl->bev.bev.timeout_write);
1143
1144                 if (bev_ssl->underlying)
1145                         consider_writing(bev_ssl);
1146         }
1147         /* XXX Handle r < 0 */
1148         (void)r;
1149 }
1150
1151
1152 static int
1153 be_openssl_enable(struct bufferevent *bev, short events)
1154 {
1155         struct bufferevent_openssl *bev_ssl = upcast(bev);
1156         int r1 = 0, r2 = 0;
1157
1158         if (events & EV_READ)
1159                 r1 = start_reading(bev_ssl);
1160         if (events & EV_WRITE)
1161                 r2 = start_writing(bev_ssl);
1162
1163         if (bev_ssl->underlying) {
1164                 if (events & EV_READ)
1165                         BEV_RESET_GENERIC_READ_TIMEOUT(bev);
1166                 if (events & EV_WRITE)
1167                         BEV_RESET_GENERIC_WRITE_TIMEOUT(bev);
1168
1169                 if (events & EV_READ)
1170                         consider_reading(bev_ssl);
1171                 if (events & EV_WRITE)
1172                         consider_writing(bev_ssl);
1173         }
1174         return (r1 < 0 || r2 < 0) ? -1 : 0;
1175 }
1176
1177 static int
1178 be_openssl_disable(struct bufferevent *bev, short events)
1179 {
1180         struct bufferevent_openssl *bev_ssl = upcast(bev);
1181
1182         if (events & EV_READ)
1183                 stop_reading(bev_ssl);
1184         if (events & EV_WRITE)
1185                 stop_writing(bev_ssl);
1186
1187         if (bev_ssl->underlying) {
1188                 if (events & EV_READ)
1189                         BEV_DEL_GENERIC_READ_TIMEOUT(bev);
1190                 if (events & EV_WRITE)
1191                         BEV_DEL_GENERIC_WRITE_TIMEOUT(bev);
1192         }
1193         return 0;
1194 }
1195
1196 static void
1197 be_openssl_unlink(struct bufferevent *bev)
1198 {
1199         struct bufferevent_openssl *bev_ssl = upcast(bev);
1200
1201         if (bev_ssl->bev.options & BEV_OPT_CLOSE_ON_FREE) {
1202                 if (bev_ssl->underlying) {
1203                         if (BEV_UPCAST(bev_ssl->underlying)->refcnt < 2) {
1204                                 event_warnx("BEV_OPT_CLOSE_ON_FREE set on an "
1205                                     "bufferevent with too few references");
1206                         } else {
1207                                 bufferevent_free(bev_ssl->underlying);
1208                                 /* We still have a reference to it, via our
1209                                  * BIO. So we don't drop this. */
1210                                 // bev_ssl->underlying = NULL;
1211                         }
1212                 }
1213         } else {
1214                 if (bev_ssl->underlying) {
1215                         if (bev_ssl->underlying->errorcb == be_openssl_eventcb)
1216                                 bufferevent_setcb(bev_ssl->underlying,
1217                                     NULL,NULL,NULL,NULL);
1218                         bufferevent_unsuspend_read_(bev_ssl->underlying,
1219                             BEV_SUSPEND_FILT_READ);
1220                 }
1221         }
1222 }
1223
1224 static void
1225 be_openssl_destruct(struct bufferevent *bev)
1226 {
1227         struct bufferevent_openssl *bev_ssl = upcast(bev);
1228
1229         if (bev_ssl->bev.options & BEV_OPT_CLOSE_ON_FREE) {
1230                 if (! bev_ssl->underlying) {
1231                         evutil_socket_t fd = -1;
1232                         BIO *bio = SSL_get_wbio(bev_ssl->ssl);
1233                         if (bio)
1234                                 fd = BIO_get_fd(bio, NULL);
1235                         if (fd >= 0)
1236                                 evutil_closesocket(fd);
1237                 }
1238                 SSL_free(bev_ssl->ssl);
1239         }
1240 }
1241
1242 static int
1243 be_openssl_adj_timeouts(struct bufferevent *bev)
1244 {
1245         struct bufferevent_openssl *bev_ssl = upcast(bev);
1246
1247         if (bev_ssl->underlying) {
1248                 return bufferevent_generic_adj_timeouts_(bev);
1249         } else {
1250                 return bufferevent_generic_adj_existing_timeouts_(bev);
1251         }
1252 }
1253
1254 static int
1255 be_openssl_flush(struct bufferevent *bufev,
1256     short iotype, enum bufferevent_flush_mode mode)
1257 {
1258         /* XXXX Implement this. */
1259         return 0;
1260 }
1261
1262 static int
1263 be_openssl_set_fd(struct bufferevent_openssl *bev_ssl,
1264     enum bufferevent_ssl_state state, int fd)
1265 {
1266         bev_ssl->state = state;
1267
1268         switch (state) {
1269         case BUFFEREVENT_SSL_ACCEPTING:
1270                 SSL_set_accept_state(bev_ssl->ssl);
1271                 if (set_handshake_callbacks(bev_ssl, fd) < 0)
1272                         return -1;
1273                 break;
1274         case BUFFEREVENT_SSL_CONNECTING:
1275                 SSL_set_connect_state(bev_ssl->ssl);
1276                 if (set_handshake_callbacks(bev_ssl, fd) < 0)
1277                         return -1;
1278                 break;
1279         case BUFFEREVENT_SSL_OPEN:
1280                 if (set_open_callbacks(bev_ssl, fd) < 0)
1281                         return -1;
1282                 break;
1283         default:
1284                 return -1;
1285         }
1286
1287         return 0;
1288 }
1289
1290 static int
1291 be_openssl_ctrl(struct bufferevent *bev,
1292     enum bufferevent_ctrl_op op, union bufferevent_ctrl_data *data)
1293 {
1294         struct bufferevent_openssl *bev_ssl = upcast(bev);
1295         switch (op) {
1296         case BEV_CTRL_SET_FD:
1297                 if (!bev_ssl->underlying) {
1298                         BIO *bio;
1299                         bio = BIO_new_socket(data->fd, 0);
1300                         SSL_set_bio(bev_ssl->ssl, bio, bio);
1301                 } else {
1302                         BIO *bio;
1303                         if (!(bio = BIO_new_bufferevent(bev_ssl->underlying, 0)))
1304                                 return -1;
1305                         SSL_set_bio(bev_ssl->ssl, bio, bio);
1306                 }
1307
1308                 return be_openssl_set_fd(bev_ssl, bev_ssl->old_state, data->fd);
1309         case BEV_CTRL_GET_FD:
1310                 if (bev_ssl->underlying) {
1311                         data->fd = event_get_fd(&bev_ssl->underlying->ev_read);
1312                 } else {
1313                         data->fd = event_get_fd(&bev->ev_read);
1314                 }
1315                 return 0;
1316         case BEV_CTRL_GET_UNDERLYING:
1317                 data->ptr = bev_ssl->underlying;
1318                 return 0;
1319         case BEV_CTRL_CANCEL_ALL:
1320         default:
1321                 return -1;
1322         }
1323 }
1324
1325 SSL *
1326 bufferevent_openssl_get_ssl(struct bufferevent *bufev)
1327 {
1328         struct bufferevent_openssl *bev_ssl = upcast(bufev);
1329         if (!bev_ssl)
1330                 return NULL;
1331         return bev_ssl->ssl;
1332 }
1333
1334 static struct bufferevent *
1335 bufferevent_openssl_new_impl(struct event_base *base,
1336     struct bufferevent *underlying,
1337     evutil_socket_t fd,
1338     SSL *ssl,
1339     enum bufferevent_ssl_state state,
1340     int options)
1341 {
1342         struct bufferevent_openssl *bev_ssl = NULL;
1343         struct bufferevent_private *bev_p = NULL;
1344         int tmp_options = options & ~BEV_OPT_THREADSAFE;
1345
1346         if (underlying != NULL && fd >= 0)
1347                 return NULL; /* Only one can be set. */
1348
1349         if (!(bev_ssl = mm_calloc(1, sizeof(struct bufferevent_openssl))))
1350                 goto err;
1351
1352         bev_p = &bev_ssl->bev;
1353
1354         if (bufferevent_init_common_(bev_p, base,
1355                 &bufferevent_ops_openssl, tmp_options) < 0)
1356                 goto err;
1357
1358         /* Don't explode if we decide to realloc a chunk we're writing from in
1359          * the output buffer. */
1360         SSL_set_mode(ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
1361
1362         bev_ssl->underlying = underlying;
1363         bev_ssl->ssl = ssl;
1364
1365         bev_ssl->outbuf_cb = evbuffer_add_cb(bev_p->bev.output,
1366             be_openssl_outbuf_cb, bev_ssl);
1367
1368         if (options & BEV_OPT_THREADSAFE)
1369                 bufferevent_enable_locking_(&bev_ssl->bev.bev, NULL);
1370
1371         if (underlying) {
1372                 bufferevent_init_generic_timeout_cbs_(&bev_ssl->bev.bev);
1373                 bufferevent_incref_(underlying);
1374         }
1375
1376         bev_ssl->old_state = state;
1377         bev_ssl->last_write = -1;
1378
1379         init_bio_counts(bev_ssl);
1380
1381         fd = be_openssl_auto_fd(bev_ssl, fd);
1382         if (be_openssl_set_fd(bev_ssl, state, fd))
1383                 goto err;
1384
1385         if (underlying) {
1386                 bufferevent_setwatermark(underlying, EV_READ, 0, 0);
1387                 bufferevent_enable(underlying, EV_READ|EV_WRITE);
1388                 if (state == BUFFEREVENT_SSL_OPEN)
1389                         bufferevent_suspend_read_(underlying,
1390                             BEV_SUSPEND_FILT_READ);
1391         }
1392
1393         return &bev_ssl->bev.bev;
1394 err:
1395         if (bev_ssl)
1396                 bufferevent_free(&bev_ssl->bev.bev);
1397         return NULL;
1398 }
1399
1400 struct bufferevent *
1401 bufferevent_openssl_filter_new(struct event_base *base,
1402     struct bufferevent *underlying,
1403     SSL *ssl,
1404     enum bufferevent_ssl_state state,
1405     int options)
1406 {
1407         /* We don't tell the BIO to close the bufferevent; we do it ourselves
1408          * on be_openssl_destruct */
1409         int close_flag = 0; /* options & BEV_OPT_CLOSE_ON_FREE; */
1410         BIO *bio;
1411         if (!underlying)
1412                 return NULL;
1413         if (!(bio = BIO_new_bufferevent(underlying, close_flag)))
1414                 return NULL;
1415
1416         SSL_set_bio(ssl, bio, bio);
1417
1418         return bufferevent_openssl_new_impl(
1419                 base, underlying, -1, ssl, state, options);
1420 }
1421
1422 struct bufferevent *
1423 bufferevent_openssl_socket_new(struct event_base *base,
1424     evutil_socket_t fd,
1425     SSL *ssl,
1426     enum bufferevent_ssl_state state,
1427     int options)
1428 {
1429         /* Does the SSL already have an fd? */
1430         BIO *bio = SSL_get_wbio(ssl);
1431         long have_fd = -1;
1432
1433         if (bio)
1434                 have_fd = BIO_get_fd(bio, NULL);
1435
1436         if (have_fd >= 0) {
1437                 /* The SSL is already configured with an fd. */
1438                 if (fd < 0) {
1439                         /* We should learn the fd from the SSL. */
1440                         fd = (evutil_socket_t) have_fd;
1441                 } else if (have_fd == (long)fd) {
1442                         /* We already know the fd from the SSL; do nothing */
1443                 } else {
1444                         /* We specified an fd different from that of the SSL.
1445                            This is probably an error on our part.  Fail. */
1446                         return NULL;
1447                 }
1448                 (void) BIO_set_close(bio, 0);
1449         } else {
1450                 /* The SSL isn't configured with a BIO with an fd. */
1451                 if (fd >= 0) {
1452                         /* ... and we have an fd we want to use. */
1453                         bio = BIO_new_socket(fd, 0);
1454                         SSL_set_bio(ssl, bio, bio);
1455                 } else {
1456                         /* Leave the fd unset. */
1457                 }
1458         }
1459
1460         return bufferevent_openssl_new_impl(
1461                 base, NULL, fd, ssl, state, options);
1462 }
1463
1464 int
1465 bufferevent_openssl_get_allow_dirty_shutdown(struct bufferevent *bev)
1466 {
1467         int allow_dirty_shutdown = -1;
1468         struct bufferevent_openssl *bev_ssl;
1469         BEV_LOCK(bev);
1470         bev_ssl = upcast(bev);
1471         if (bev_ssl)
1472                 allow_dirty_shutdown = bev_ssl->allow_dirty_shutdown;
1473         BEV_UNLOCK(bev);
1474         return allow_dirty_shutdown;
1475 }
1476
1477 void
1478 bufferevent_openssl_set_allow_dirty_shutdown(struct bufferevent *bev,
1479     int allow_dirty_shutdown)
1480 {
1481         struct bufferevent_openssl *bev_ssl;
1482         BEV_LOCK(bev);
1483         bev_ssl = upcast(bev);
1484         if (bev_ssl)
1485                 bev_ssl->allow_dirty_shutdown = !!allow_dirty_shutdown;
1486         BEV_UNLOCK(bev);
1487 }
1488
1489 unsigned long
1490 bufferevent_get_openssl_error(struct bufferevent *bev)
1491 {
1492         unsigned long err = 0;
1493         struct bufferevent_openssl *bev_ssl;
1494         BEV_LOCK(bev);
1495         bev_ssl = upcast(bev);
1496         if (bev_ssl && bev_ssl->n_errors) {
1497                 err = bev_ssl->errors[--bev_ssl->n_errors];
1498         }
1499         BEV_UNLOCK(bev);
1500         return err;
1501 }