contrib/unbound/util/netevent.c

   1 /*
   2  * util/netevent.c - event notification
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains event notification functions.
  40  */
  41 #include "config.h"
  42 #include "util/netevent.h"
  43 #include "util/ub_event.h"
  44 #include "util/log.h"
  45 #include "util/net_help.h"
  46 #include "util/tcp_conn_limit.h"
  47 #include "util/fptr_wlist.h"
  48 #include "sldns/pkthdr.h"
  49 #include "sldns/sbuffer.h"
  50 #include "sldns/str2wire.h"
  51 #include "dnstap/dnstap.h"
  52 #include "dnscrypt/dnscrypt.h"
  53 #include "services/listen_dnsport.h"
  54 #ifdef HAVE_OPENSSL_SSL_H
  55 #include <openssl/ssl.h>
  56 #endif
  57 #ifdef HAVE_OPENSSL_ERR_H
  58 #include <openssl/err.h>
  59 #endif
  60
  61 /* -------- Start of local definitions -------- */
  62 /** if CMSG_ALIGN is not defined on this platform, a workaround */
  63 #ifndef CMSG_ALIGN
  64 #  ifdef __CMSG_ALIGN
  65 #    define CMSG_ALIGN(n) __CMSG_ALIGN(n)
  66 #  elif defined(CMSG_DATA_ALIGN)
  67 #    define CMSG_ALIGN _CMSG_DATA_ALIGN
  68 #  else
  69 #    define CMSG_ALIGN(len) (((len)+sizeof(long)-1) & ~(sizeof(long)-1))
  70 #  endif
  71 #endif
  72
  73 /** if CMSG_LEN is not defined on this platform, a workaround */
  74 #ifndef CMSG_LEN
  75 #  define CMSG_LEN(len) (CMSG_ALIGN(sizeof(struct cmsghdr))+(len))
  76 #endif
  77
  78 /** if CMSG_SPACE is not defined on this platform, a workaround */
  79 #ifndef CMSG_SPACE
  80 #  ifdef _CMSG_HDR_ALIGN
  81 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+_CMSG_HDR_ALIGN(sizeof(struct cmsghdr)))
  82 #  else
  83 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+CMSG_ALIGN(sizeof(struct cmsghdr)))
  84 #  endif
  85 #endif
  86
  87 /** The TCP writing query timeout in milliseconds */
  88 #define TCP_QUERY_TIMEOUT 120000
  89 /** The minimum actual TCP timeout to use, regardless of what we advertise,
  90  * in msec */
  91 #define TCP_QUERY_TIMEOUT_MINIMUM 200
  92
  93 #ifndef NONBLOCKING_IS_BROKEN
  94 /** number of UDP reads to perform per read indication from select */
  95 #define NUM_UDP_PER_SELECT 100
  96 #else
  97 #define NUM_UDP_PER_SELECT 1
  98 #endif
  99
 100 /**
 101  * The internal event structure for keeping ub_event info for the event.
 102  * Possibly other structures (list, tree) this is part of.
 103  */
 104 struct internal_event {
 105         /** the comm base */
 106         struct comm_base* base;
 107         /** ub_event event type */
 108         struct ub_event* ev;
 109 };
 110
 111 /**
 112  * Internal base structure, so that every thread has its own events.
 113  */
 114 struct internal_base {
 115         /** ub_event event_base type. */
 116         struct ub_event_base* base;
 117         /** seconds time pointer points here */
 118         time_t secs;
 119         /** timeval with current time */
 120         struct timeval now;
 121         /** the event used for slow_accept timeouts */
 122         struct ub_event* slow_accept;
 123         /** true if slow_accept is enabled */
 124         int slow_accept_enabled;
 125 };
 126
 127 /**
 128  * Internal timer structure, to store timer event in.
 129  */
 130 struct internal_timer {
 131         /** the super struct from which derived */
 132         struct comm_timer super;
 133         /** the comm base */
 134         struct comm_base* base;
 135         /** ub_event event type */
 136         struct ub_event* ev;
 137         /** is timer enabled */
 138         uint8_t enabled;
 139 };
 140
 141 /**
 142  * Internal signal structure, to store signal event in.
 143  */
 144 struct internal_signal {
 145         /** ub_event event type */
 146         struct ub_event* ev;
 147         /** next in signal list */
 148         struct internal_signal* next;
 149 };
 150
 151 /** create a tcp handler with a parent */
 152 static struct comm_point* comm_point_create_tcp_handler(
 153         struct comm_base *base, struct comm_point* parent, size_t bufsize,
 154         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
 155         void* callback_arg);
 156
 157 /* -------- End of local definitions -------- */
 158
 159 struct comm_base*
 160 comm_base_create(int sigs)
 161 {
 162         struct comm_base* b = (struct comm_base*)calloc(1,
 163                 sizeof(struct comm_base));
 164         const char *evnm="event", *evsys="", *evmethod="";
 165
 166         if(!b)
 167                 return NULL;
 168         b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
 169         if(!b->eb) {
 170                 free(b);
 171                 return NULL;
 172         }
 173         b->eb->base = ub_default_event_base(sigs, &b->eb->secs, &b->eb->now);
 174         if(!b->eb->base) {
 175                 free(b->eb);
 176                 free(b);
 177                 return NULL;
 178         }
 179         ub_comm_base_now(b);
 180         ub_get_event_sys(b->eb->base, &evnm, &evsys, &evmethod);
 181         verbose(VERB_ALGO, "%s %s uses %s method.", evnm, evsys, evmethod);
 182         return b;
 183 }
 184
 185 struct comm_base*
 186 comm_base_create_event(struct ub_event_base* base)
 187 {
 188         struct comm_base* b = (struct comm_base*)calloc(1,
 189                 sizeof(struct comm_base));
 190         if(!b)
 191                 return NULL;
 192         b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
 193         if(!b->eb) {
 194                 free(b);
 195                 return NULL;
 196         }
 197         b->eb->base = base;
 198         ub_comm_base_now(b);
 199         return b;
 200 }
 201
 202 void
 203 comm_base_delete(struct comm_base* b)
 204 {
 205         if(!b)
 206                 return;
 207         if(b->eb->slow_accept_enabled) {
 208                 if(ub_event_del(b->eb->slow_accept) != 0) {
 209                         log_err("could not event_del slow_accept");
 210                 }
 211                 ub_event_free(b->eb->slow_accept);
 212         }
 213         ub_event_base_free(b->eb->base);
 214         b->eb->base = NULL;
 215         free(b->eb);
 216         free(b);
 217 }
 218
 219 void
 220 comm_base_delete_no_base(struct comm_base* b)
 221 {
 222         if(!b)
 223                 return;
 224         if(b->eb->slow_accept_enabled) {
 225                 if(ub_event_del(b->eb->slow_accept) != 0) {
 226                         log_err("could not event_del slow_accept");
 227                 }
 228                 ub_event_free(b->eb->slow_accept);
 229         }
 230         b->eb->base = NULL;
 231         free(b->eb);
 232         free(b);
 233 }
 234
 235 void
 236 comm_base_timept(struct comm_base* b, time_t** tt, struct timeval** tv)
 237 {
 238         *tt = &b->eb->secs;
 239         *tv = &b->eb->now;
 240 }
 241
 242 void
 243 comm_base_dispatch(struct comm_base* b)
 244 {
 245         int retval;
 246         retval = ub_event_base_dispatch(b->eb->base);
 247         if(retval < 0) {
 248                 fatal_exit("event_dispatch returned error %d, "
 249                         "errno is %s", retval, strerror(errno));
 250         }
 251 }
 252
 253 void comm_base_exit(struct comm_base* b)
 254 {
 255         if(ub_event_base_loopexit(b->eb->base) != 0) {
 256                 log_err("Could not loopexit");
 257         }
 258 }
 259
 260 void comm_base_set_slow_accept_handlers(struct comm_base* b,
 261         void (*stop_acc)(void*), void (*start_acc)(void*), void* arg)
 262 {
 263         b->stop_accept = stop_acc;
 264         b->start_accept = start_acc;
 265         b->cb_arg = arg;
 266 }
 267
 268 struct ub_event_base* comm_base_internal(struct comm_base* b)
 269 {
 270         return b->eb->base;
 271 }
 272
 273 /** see if errno for udp has to be logged or not uses globals */
 274 static int
 275 udp_send_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
 276 {
 277         /* do not log transient errors (unless high verbosity) */
 278 #if defined(ENETUNREACH) || defined(EHOSTDOWN) || defined(EHOSTUNREACH) || defined(ENETDOWN)
 279         switch(errno) {
 280 #  ifdef ENETUNREACH
 281                 case ENETUNREACH:
 282 #  endif
 283 #  ifdef EHOSTDOWN
 284                 case EHOSTDOWN:
 285 #  endif
 286 #  ifdef EHOSTUNREACH
 287                 case EHOSTUNREACH:
 288 #  endif
 289 #  ifdef ENETDOWN
 290                 case ENETDOWN:
 291 #  endif
 292                         if(verbosity < VERB_ALGO)
 293                                 return 0;
 294                 default:
 295                         break;
 296         }
 297 #endif
 298         /* permission denied is gotten for every send if the
 299          * network is disconnected (on some OS), squelch it */
 300         if( ((errno == EPERM)
 301 #  ifdef EADDRNOTAVAIL
 302                 /* 'Cannot assign requested address' also when disconnected */
 303                 || (errno == EADDRNOTAVAIL)
 304 #  endif
 305                 ) && verbosity < VERB_DETAIL)
 306                 return 0;
 307 #  ifdef EADDRINUSE
 308         /* If SO_REUSEADDR is set, we could try to connect to the same server
 309          * from the same source port twice. */
 310         if(errno == EADDRINUSE && verbosity < VERB_DETAIL)
 311                 return 0;
 312 #  endif
 313         /* squelch errors where people deploy AAAA ::ffff:bla for
 314          * authority servers, which we try for intranets. */
 315         if(errno == EINVAL && addr_is_ip4mapped(
 316                 (struct sockaddr_storage*)addr, addrlen) &&
 317                 verbosity < VERB_DETAIL)
 318                 return 0;
 319         /* SO_BROADCAST sockopt can give access to 255.255.255.255,
 320          * but a dns cache does not need it. */
 321         if(errno == EACCES && addr_is_broadcast(
 322                 (struct sockaddr_storage*)addr, addrlen) &&
 323                 verbosity < VERB_DETAIL)
 324                 return 0;
 325         return 1;
 326 }
 327
 328 int tcp_connect_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
 329 {
 330         return udp_send_errno_needs_log(addr, addrlen);
 331 }
 332
 333 /* send a UDP reply */
 334 int
 335 comm_point_send_udp_msg(struct comm_point *c, sldns_buffer* packet,
 336         struct sockaddr* addr, socklen_t addrlen)
 337 {
 338         ssize_t sent;
 339         log_assert(c->fd != -1);
 340 #ifdef UNBOUND_DEBUG
 341         if(sldns_buffer_remaining(packet) == 0)
 342                 log_err("error: send empty UDP packet");
 343 #endif
 344         log_assert(addr && addrlen > 0);
 345         sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
 346                 sldns_buffer_remaining(packet), 0,
 347                 addr, addrlen);
 348         if(sent == -1) {
 349                 /* try again and block, waiting for IO to complete,
 350                  * we want to send the answer, and we will wait for
 351                  * the ethernet interface buffer to have space. */
 352 #ifndef USE_WINSOCK
 353                 if(errno == EAGAIN ||
 354 #  ifdef EWOULDBLOCK
 355                         errno == EWOULDBLOCK ||
 356 #  endif
 357                         errno == ENOBUFS) {
 358 #else
 359                 if(WSAGetLastError() == WSAEINPROGRESS ||
 360                         WSAGetLastError() == WSAENOBUFS ||
 361                         WSAGetLastError() == WSAEWOULDBLOCK) {
 362 #endif
 363                         int e;
 364                         fd_set_block(c->fd);
 365                         sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
 366                                 sldns_buffer_remaining(packet), 0,
 367                                 addr, addrlen);
 368                         e = errno;
 369                         fd_set_nonblock(c->fd);
 370                         errno = e;
 371                 }
 372         }
 373         if(sent == -1) {
 374                 if(!udp_send_errno_needs_log(addr, addrlen))
 375                         return 0;
 376 #ifndef USE_WINSOCK
 377                 verbose(VERB_OPS, "sendto failed: %s", strerror(errno));
 378 #else
 379                 verbose(VERB_OPS, "sendto failed: %s",
 380                         wsa_strerror(WSAGetLastError()));
 381 #endif
 382                 log_addr(VERB_OPS, "remote address is",
 383                         (struct sockaddr_storage*)addr, addrlen);
 384                 return 0;
 385         } else if((size_t)sent != sldns_buffer_remaining(packet)) {
 386                 log_err("sent %d in place of %d bytes",
 387                         (int)sent, (int)sldns_buffer_remaining(packet));
 388                 return 0;
 389         }
 390         return 1;
 391 }
 392
 393 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && (defined(HAVE_RECVMSG) || defined(HAVE_SENDMSG))
 394 /** print debug ancillary info */
 395 static void p_ancil(const char* str, struct comm_reply* r)
 396 {
 397         if(r->srctype != 4 && r->srctype != 6) {
 398                 log_info("%s: unknown srctype %d", str, r->srctype);
 399                 return;
 400         }
 401         if(r->srctype == 6) {
 402                 char buf[1024];
 403                 if(inet_ntop(AF_INET6, &r->pktinfo.v6info.ipi6_addr,
 404                         buf, (socklen_t)sizeof(buf)) == 0) {
 405                         (void)strlcpy(buf, "(inet_ntop error)", sizeof(buf));
 406                 }
 407                 buf[sizeof(buf)-1]=0;
 408                 log_info("%s: %s %d", str, buf, r->pktinfo.v6info.ipi6_ifindex);
 409         } else if(r->srctype == 4) {
 410 #ifdef IP_PKTINFO
 411                 char buf1[1024], buf2[1024];
 412                 if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_addr,
 413                         buf1, (socklen_t)sizeof(buf1)) == 0) {
 414                         (void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
 415                 }
 416                 buf1[sizeof(buf1)-1]=0;
 417 #ifdef HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST
 418                 if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_spec_dst,
 419                         buf2, (socklen_t)sizeof(buf2)) == 0) {
 420                         (void)strlcpy(buf2, "(inet_ntop error)", sizeof(buf2));
 421                 }
 422                 buf2[sizeof(buf2)-1]=0;
 423 #else
 424                 buf2[0]=0;
 425 #endif
 426                 log_info("%s: %d %s %s", str, r->pktinfo.v4info.ipi_ifindex,
 427                         buf1, buf2);
 428 #elif defined(IP_RECVDSTADDR)
 429                 char buf1[1024];
 430                 if(inet_ntop(AF_INET, &r->pktinfo.v4addr,
 431                         buf1, (socklen_t)sizeof(buf1)) == 0) {
 432                         (void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
 433                 }
 434                 buf1[sizeof(buf1)-1]=0;
 435                 log_info("%s: %s", str, buf1);
 436 #endif /* IP_PKTINFO or PI_RECVDSTDADDR */
 437         }
 438 }
 439 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG||HAVE_SENDMSG */
 440
 441 /** send a UDP reply over specified interface*/
 442 static int
 443 comm_point_send_udp_msg_if(struct comm_point *c, sldns_buffer* packet,
 444         struct sockaddr* addr, socklen_t addrlen, struct comm_reply* r)
 445 {
 446 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_SENDMSG)
 447         ssize_t sent;
 448         struct msghdr msg;
 449         struct iovec iov[1];
 450         char control[256];
 451 #ifndef S_SPLINT_S
 452         struct cmsghdr *cmsg;
 453 #endif /* S_SPLINT_S */
 454
 455         log_assert(c->fd != -1);
 456 #ifdef UNBOUND_DEBUG
 457         if(sldns_buffer_remaining(packet) == 0)
 458                 log_err("error: send empty UDP packet");
 459 #endif
 460         log_assert(addr && addrlen > 0);
 461
 462         msg.msg_name = addr;
 463         msg.msg_namelen = addrlen;
 464         iov[0].iov_base = sldns_buffer_begin(packet);
 465         iov[0].iov_len = sldns_buffer_remaining(packet);
 466         msg.msg_iov = iov;
 467         msg.msg_iovlen = 1;
 468         msg.msg_control = control;
 469 #ifndef S_SPLINT_S
 470         msg.msg_controllen = sizeof(control);
 471 #endif /* S_SPLINT_S */
 472         msg.msg_flags = 0;
 473
 474 #ifndef S_SPLINT_S
 475         cmsg = CMSG_FIRSTHDR(&msg);
 476         if(r->srctype == 4) {
 477 #ifdef IP_PKTINFO
 478                 void* cmsg_data;
 479                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo));
 480                 log_assert(msg.msg_controllen <= sizeof(control));
 481                 cmsg->cmsg_level = IPPROTO_IP;
 482                 cmsg->cmsg_type = IP_PKTINFO;
 483                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v4info,
 484                         sizeof(struct in_pktinfo));
 485                 /* unset the ifindex to not bypass the routing tables */
 486                 cmsg_data = CMSG_DATA(cmsg);
 487                 ((struct in_pktinfo *) cmsg_data)->ipi_ifindex = 0;
 488                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
 489 #elif defined(IP_SENDSRCADDR)
 490                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in_addr));
 491                 log_assert(msg.msg_controllen <= sizeof(control));
 492                 cmsg->cmsg_level = IPPROTO_IP;
 493                 cmsg->cmsg_type = IP_SENDSRCADDR;
 494                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v4addr,
 495                         sizeof(struct in_addr));
 496                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_addr));
 497 #else
 498                 verbose(VERB_ALGO, "no IP_PKTINFO or IP_SENDSRCADDR");
 499                 msg.msg_control = NULL;
 500 #endif /* IP_PKTINFO or IP_SENDSRCADDR */
 501         } else if(r->srctype == 6) {
 502                 void* cmsg_data;
 503                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
 504                 log_assert(msg.msg_controllen <= sizeof(control));
 505                 cmsg->cmsg_level = IPPROTO_IPV6;
 506                 cmsg->cmsg_type = IPV6_PKTINFO;
 507                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v6info,
 508                         sizeof(struct in6_pktinfo));
 509                 /* unset the ifindex to not bypass the routing tables */
 510                 cmsg_data = CMSG_DATA(cmsg);
 511                 ((struct in6_pktinfo *) cmsg_data)->ipi6_ifindex = 0;
 512                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
 513         } else {
 514                 /* try to pass all 0 to use default route */
 515                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
 516                 log_assert(msg.msg_controllen <= sizeof(control));
 517                 cmsg->cmsg_level = IPPROTO_IPV6;
 518                 cmsg->cmsg_type = IPV6_PKTINFO;
 519                 memset(CMSG_DATA(cmsg), 0, sizeof(struct in6_pktinfo));
 520                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
 521         }
 522 #endif /* S_SPLINT_S */
 523         if(verbosity >= VERB_ALGO)
 524                 p_ancil("send_udp over interface", r);
 525         sent = sendmsg(c->fd, &msg, 0);
 526         if(sent == -1) {
 527                 /* try again and block, waiting for IO to complete,
 528                  * we want to send the answer, and we will wait for
 529                  * the ethernet interface buffer to have space. */
 530 #ifndef USE_WINSOCK
 531                 if(errno == EAGAIN ||
 532 #  ifdef EWOULDBLOCK
 533                         errno == EWOULDBLOCK ||
 534 #  endif
 535                         errno == ENOBUFS) {
 536 #else
 537                 if(WSAGetLastError() == WSAEINPROGRESS ||
 538                         WSAGetLastError() == WSAENOBUFS ||
 539                         WSAGetLastError() == WSAEWOULDBLOCK) {
 540 #endif
 541                         int e;
 542                         fd_set_block(c->fd);
 543                         sent = sendmsg(c->fd, &msg, 0);
 544                         e = errno;
 545                         fd_set_nonblock(c->fd);
 546                         errno = e;
 547                 }
 548         }
 549         if(sent == -1) {
 550                 if(!udp_send_errno_needs_log(addr, addrlen))
 551                         return 0;
 552                 verbose(VERB_OPS, "sendmsg failed: %s", strerror(errno));
 553                 log_addr(VERB_OPS, "remote address is",
 554                         (struct sockaddr_storage*)addr, addrlen);
 555 #ifdef __NetBSD__
 556                 /* netbsd 7 has IP_PKTINFO for recv but not send */
 557                 if(errno == EINVAL && r->srctype == 4)
 558                         log_err("sendmsg: No support for sendmsg(IP_PKTINFO). "
 559                                 "Please disable interface-automatic");
 560 #endif
 561                 return 0;
 562         } else if((size_t)sent != sldns_buffer_remaining(packet)) {
 563                 log_err("sent %d in place of %d bytes",
 564                         (int)sent, (int)sldns_buffer_remaining(packet));
 565                 return 0;
 566         }
 567         return 1;
 568 #else
 569         (void)c;
 570         (void)packet;
 571         (void)addr;
 572         (void)addrlen;
 573         (void)r;
 574         log_err("sendmsg: IPV6_PKTINFO not supported");
 575         return 0;
 576 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_SENDMSG */
 577 }
 578
 579 void
 580 comm_point_udp_ancil_callback(int fd, short event, void* arg)
 581 {
 582 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_RECVMSG)
 583         struct comm_reply rep;
 584         struct msghdr msg;
 585         struct iovec iov[1];
 586         ssize_t rcv;
 587         char ancil[256];
 588         int i;
 589 #ifndef S_SPLINT_S
 590         struct cmsghdr* cmsg;
 591 #endif /* S_SPLINT_S */
 592
 593         rep.c = (struct comm_point*)arg;
 594         log_assert(rep.c->type == comm_udp);
 595
 596         if(!(event&UB_EV_READ))
 597                 return;
 598         log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
 599         ub_comm_base_now(rep.c->ev->base);
 600         for(i=0; i<NUM_UDP_PER_SELECT; i++) {
 601                 sldns_buffer_clear(rep.c->buffer);
 602                 rep.addrlen = (socklen_t)sizeof(rep.addr);
 603                 log_assert(fd != -1);
 604                 log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
 605                 msg.msg_name = &rep.addr;
 606                 msg.msg_namelen = (socklen_t)sizeof(rep.addr);
 607                 iov[0].iov_base = sldns_buffer_begin(rep.c->buffer);
 608                 iov[0].iov_len = sldns_buffer_remaining(rep.c->buffer);
 609                 msg.msg_iov = iov;
 610                 msg.msg_iovlen = 1;
 611                 msg.msg_control = ancil;
 612 #ifndef S_SPLINT_S
 613                 msg.msg_controllen = sizeof(ancil);
 614 #endif /* S_SPLINT_S */
 615                 msg.msg_flags = 0;
 616                 rcv = recvmsg(fd, &msg, 0);
 617                 if(rcv == -1) {
 618                         if(errno != EAGAIN && errno != EINTR) {
 619                                 log_err("recvmsg failed: %s", strerror(errno));
 620                         }
 621                         return;
 622                 }
 623                 rep.addrlen = msg.msg_namelen;
 624                 sldns_buffer_skip(rep.c->buffer, rcv);
 625                 sldns_buffer_flip(rep.c->buffer);
 626                 rep.srctype = 0;
 627 #ifndef S_SPLINT_S
 628                 for(cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
 629                         cmsg = CMSG_NXTHDR(&msg, cmsg)) {
 630                         if( cmsg->cmsg_level == IPPROTO_IPV6 &&
 631                                 cmsg->cmsg_type == IPV6_PKTINFO) {
 632                                 rep.srctype = 6;
 633                                 memmove(&rep.pktinfo.v6info, CMSG_DATA(cmsg),
 634                                         sizeof(struct in6_pktinfo));
 635                                 break;
 636 #ifdef IP_PKTINFO
 637                         } else if( cmsg->cmsg_level == IPPROTO_IP &&
 638                                 cmsg->cmsg_type == IP_PKTINFO) {
 639                                 rep.srctype = 4;
 640                                 memmove(&rep.pktinfo.v4info, CMSG_DATA(cmsg),
 641                                         sizeof(struct in_pktinfo));
 642                                 break;
 643 #elif defined(IP_RECVDSTADDR)
 644                         } else if( cmsg->cmsg_level == IPPROTO_IP &&
 645                                 cmsg->cmsg_type == IP_RECVDSTADDR) {
 646                                 rep.srctype = 4;
 647                                 memmove(&rep.pktinfo.v4addr, CMSG_DATA(cmsg),
 648                                         sizeof(struct in_addr));
 649                                 break;
 650 #endif /* IP_PKTINFO or IP_RECVDSTADDR */
 651                         }
 652                 }
 653                 if(verbosity >= VERB_ALGO)
 654                         p_ancil("receive_udp on interface", &rep);
 655 #endif /* S_SPLINT_S */
 656                 fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
 657                 if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
 658                         /* send back immediate reply */
 659                         (void)comm_point_send_udp_msg_if(rep.c, rep.c->buffer,
 660                                 (struct sockaddr*)&rep.addr, rep.addrlen, &rep);
 661                 }
 662                 if(!rep.c || rep.c->fd == -1) /* commpoint closed */
 663                         break;
 664         }
 665 #else
 666         (void)fd;
 667         (void)event;
 668         (void)arg;
 669         fatal_exit("recvmsg: No support for IPV6_PKTINFO; IP_PKTINFO or IP_RECVDSTADDR. "
 670                 "Please disable interface-automatic");
 671 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG */
 672 }
 673
 674 void
 675 comm_point_udp_callback(int fd, short event, void* arg)
 676 {
 677         struct comm_reply rep;
 678         ssize_t rcv;
 679         int i;
 680         struct sldns_buffer *buffer;
 681
 682         rep.c = (struct comm_point*)arg;
 683         log_assert(rep.c->type == comm_udp);
 684
 685         if(!(event&UB_EV_READ))
 686                 return;
 687         log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
 688         ub_comm_base_now(rep.c->ev->base);
 689         for(i=0; i<NUM_UDP_PER_SELECT; i++) {
 690                 sldns_buffer_clear(rep.c->buffer);
 691                 rep.addrlen = (socklen_t)sizeof(rep.addr);
 692                 log_assert(fd != -1);
 693                 log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
 694                 rcv = recvfrom(fd, (void*)sldns_buffer_begin(rep.c->buffer),
 695                         sldns_buffer_remaining(rep.c->buffer), 0,
 696                         (struct sockaddr*)&rep.addr, &rep.addrlen);
 697                 if(rcv == -1) {
 698 #ifndef USE_WINSOCK
 699                         if(errno != EAGAIN && errno != EINTR)
 700                                 log_err("recvfrom %d failed: %s",
 701                                         fd, strerror(errno));
 702 #else
 703                         if(WSAGetLastError() != WSAEINPROGRESS &&
 704                                 WSAGetLastError() != WSAECONNRESET &&
 705                                 WSAGetLastError()!= WSAEWOULDBLOCK)
 706                                 log_err("recvfrom failed: %s",
 707                                         wsa_strerror(WSAGetLastError()));
 708 #endif
 709                         return;
 710                 }
 711                 sldns_buffer_skip(rep.c->buffer, rcv);
 712                 sldns_buffer_flip(rep.c->buffer);
 713                 rep.srctype = 0;
 714                 fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
 715                 if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
 716                         /* send back immediate reply */
 717 #ifdef USE_DNSCRYPT
 718                         buffer = rep.c->dnscrypt_buffer;
 719 #else
 720                         buffer = rep.c->buffer;
 721 #endif
 722                         (void)comm_point_send_udp_msg(rep.c, buffer,
 723                                 (struct sockaddr*)&rep.addr, rep.addrlen);
 724                 }
 725                 if(!rep.c || rep.c->fd != fd) /* commpoint closed to -1 or reused for
 726                 another UDP port. Note rep.c cannot be reused with TCP fd. */
 727                         break;
 728         }
 729 }
 730
 731 /** Use a new tcp handler for new query fd, set to read query */
 732 static void
 733 setup_tcp_handler(struct comm_point* c, int fd, int cur, int max)
 734 {
 735         int handler_usage;
 736         log_assert(c->type == comm_tcp);
 737         log_assert(c->fd == -1);
 738         sldns_buffer_clear(c->buffer);
 739 #ifdef USE_DNSCRYPT
 740         if (c->dnscrypt)
 741                 sldns_buffer_clear(c->dnscrypt_buffer);
 742 #endif
 743         c->tcp_is_reading = 1;
 744         c->tcp_byte_count = 0;
 745         /* if more than half the tcp handlers are in use, use a shorter
 746          * timeout for this TCP connection, we need to make space for
 747          * other connections to be able to get attention */
 748         /* If > 50% TCP handler structures in use, set timeout to 1/100th
 749          *      configured value.
 750          * If > 65%TCP handler structures in use, set to 1/500th configured
 751          *      value.
 752          * If > 80% TCP handler structures in use, set to 0.
 753          *
 754          * If the timeout to use falls below 200 milliseconds, an actual
 755          * timeout of 200ms is used.
 756          */
 757         handler_usage = (cur * 100) / max;
 758         if(handler_usage > 50 && handler_usage <= 65)
 759                 c->tcp_timeout_msec /= 100;
 760         else if (handler_usage > 65 && handler_usage <= 80)
 761                 c->tcp_timeout_msec /= 500;
 762         else if (handler_usage > 80)
 763                 c->tcp_timeout_msec = 0;
 764         comm_point_start_listening(c, fd,
 765                 c->tcp_timeout_msec < TCP_QUERY_TIMEOUT_MINIMUM
 766                         ? TCP_QUERY_TIMEOUT_MINIMUM
 767                         : c->tcp_timeout_msec);
 768 }
 769
 770 void comm_base_handle_slow_accept(int ATTR_UNUSED(fd),
 771         short ATTR_UNUSED(event), void* arg)
 772 {
 773         struct comm_base* b = (struct comm_base*)arg;
 774         /* timeout for the slow accept, re-enable accepts again */
 775         if(b->start_accept) {
 776                 verbose(VERB_ALGO, "wait is over, slow accept disabled");
 777                 fptr_ok(fptr_whitelist_start_accept(b->start_accept));
 778                 (*b->start_accept)(b->cb_arg);
 779                 b->eb->slow_accept_enabled = 0;
 780         }
 781 }
 782
 783 int comm_point_perform_accept(struct comm_point* c,
 784         struct sockaddr_storage* addr, socklen_t* addrlen)
 785 {
 786         int new_fd;
 787         *addrlen = (socklen_t)sizeof(*addr);
 788 #ifndef HAVE_ACCEPT4
 789         new_fd = accept(c->fd, (struct sockaddr*)addr, addrlen);
 790 #else
 791         /* SOCK_NONBLOCK saves extra calls to fcntl for the same result */
 792         new_fd = accept4(c->fd, (struct sockaddr*)addr, addrlen, SOCK_NONBLOCK);
 793 #endif
 794         if(new_fd == -1) {
 795 #ifndef USE_WINSOCK
 796                 /* EINTR is signal interrupt. others are closed connection. */
 797                 if(     errno == EINTR || errno == EAGAIN
 798 #ifdef EWOULDBLOCK
 799                         || errno == EWOULDBLOCK
 800 #endif
 801 #ifdef ECONNABORTED
 802                         || errno == ECONNABORTED
 803 #endif
 804 #ifdef EPROTO
 805                         || errno == EPROTO
 806 #endif /* EPROTO */
 807                         )
 808                         return -1;
 809 #if defined(ENFILE) && defined(EMFILE)
 810                 if(errno == ENFILE || errno == EMFILE) {
 811                         /* out of file descriptors, likely outside of our
 812                          * control. stop accept() calls for some time */
 813                         if(c->ev->base->stop_accept) {
 814                                 struct comm_base* b = c->ev->base;
 815                                 struct timeval tv;
 816                                 verbose(VERB_ALGO, "out of file descriptors: "
 817                                         "slow accept");
 818                                 b->eb->slow_accept_enabled = 1;
 819                                 fptr_ok(fptr_whitelist_stop_accept(
 820                                         b->stop_accept));
 821                                 (*b->stop_accept)(b->cb_arg);
 822                                 /* set timeout, no mallocs */
 823                                 tv.tv_sec = NETEVENT_SLOW_ACCEPT_TIME/1000;
 824                                 tv.tv_usec = (NETEVENT_SLOW_ACCEPT_TIME%1000)*1000;
 825                                 b->eb->slow_accept = ub_event_new(b->eb->base,
 826                                         -1, UB_EV_TIMEOUT,
 827                                         comm_base_handle_slow_accept, b);
 828                                 if(b->eb->slow_accept == NULL) {
 829                                         /* we do not want to log here, because
 830                                          * that would spam the logfiles.
 831                                          * error: "event_base_set failed." */
 832                                 }
 833                                 else if(ub_event_add(b->eb->slow_accept, &tv)
 834                                         != 0) {
 835                                         /* we do not want to log here,
 836                                          * error: "event_add failed." */
 837                                 }
 838                         }
 839                         return -1;
 840                 }
 841 #endif
 842                 log_err_addr("accept failed", strerror(errno), addr, *addrlen);
 843 #else /* USE_WINSOCK */
 844                 if(WSAGetLastError() == WSAEINPROGRESS ||
 845                         WSAGetLastError() == WSAECONNRESET)
 846                         return -1;
 847                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
 848                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
 849                         return -1;
 850                 }
 851                 log_err_addr("accept failed", wsa_strerror(WSAGetLastError()),
 852                         addr, *addrlen);
 853 #endif
 854                 return -1;
 855         }
 856         if(c->tcp_conn_limit && c->type == comm_tcp_accept) {
 857                 c->tcl_addr = tcl_addr_lookup(c->tcp_conn_limit, addr, *addrlen);
 858                 if(!tcl_new_connection(c->tcl_addr)) {
 859                         if(verbosity >= 3)
 860                                 log_err_addr("accept rejected",
 861                                 "connection limit exceeded", addr, *addrlen);
 862                         close(new_fd);
 863                         return -1;
 864                 }
 865         }
 866 #ifndef HAVE_ACCEPT4
 867         fd_set_nonblock(new_fd);
 868 #endif
 869         return new_fd;
 870 }
 871
 872 #ifdef USE_WINSOCK
 873 static long win_bio_cb(BIO *b, int oper, const char* ATTR_UNUSED(argp),
 874         int ATTR_UNUSED(argi), long argl, long retvalue)
 875 {
 876         int wsa_err = WSAGetLastError(); /* store errcode before it is gone */
 877         verbose(VERB_ALGO, "bio_cb %d, %s %s %s", oper,
 878                 (oper&BIO_CB_RETURN)?"return":"before",
 879                 (oper&BIO_CB_READ)?"read":((oper&BIO_CB_WRITE)?"write":"other"),
 880                 wsa_err==WSAEWOULDBLOCK?"wsawb":"");
 881         /* on windows, check if previous operation caused EWOULDBLOCK */
 882         if( (oper == (BIO_CB_READ|BIO_CB_RETURN) && argl == 0) ||
 883                 (oper == (BIO_CB_GETS|BIO_CB_RETURN) && argl == 0)) {
 884                 if(wsa_err == WSAEWOULDBLOCK)
 885                         ub_winsock_tcp_wouldblock((struct ub_event*)
 886                                 BIO_get_callback_arg(b), UB_EV_READ);
 887         }
 888         if( (oper == (BIO_CB_WRITE|BIO_CB_RETURN) && argl == 0) ||
 889                 (oper == (BIO_CB_PUTS|BIO_CB_RETURN) && argl == 0)) {
 890                 if(wsa_err == WSAEWOULDBLOCK)
 891                         ub_winsock_tcp_wouldblock((struct ub_event*)
 892                                 BIO_get_callback_arg(b), UB_EV_WRITE);
 893         }
 894         /* return original return value */
 895         return retvalue;
 896 }
 897
 898 /** set win bio callbacks for nonblocking operations */
 899 void
 900 comm_point_tcp_win_bio_cb(struct comm_point* c, void* thessl)
 901 {
 902         SSL* ssl = (SSL*)thessl;
 903         /* set them both just in case, but usually they are the same BIO */
 904         BIO_set_callback(SSL_get_rbio(ssl), &win_bio_cb);
 905         BIO_set_callback_arg(SSL_get_rbio(ssl), (char*)c->ev->ev);
 906         BIO_set_callback(SSL_get_wbio(ssl), &win_bio_cb);
 907         BIO_set_callback_arg(SSL_get_wbio(ssl), (char*)c->ev->ev);
 908 }
 909 #endif
 910
 911 void
 912 comm_point_tcp_accept_callback(int fd, short event, void* arg)
 913 {
 914         struct comm_point* c = (struct comm_point*)arg, *c_hdl;
 915         int new_fd;
 916         log_assert(c->type == comm_tcp_accept);
 917         if(!(event & UB_EV_READ)) {
 918                 log_info("ignoring tcp accept event %d", (int)event);
 919                 return;
 920         }
 921         ub_comm_base_now(c->ev->base);
 922         /* find free tcp handler. */
 923         if(!c->tcp_free) {
 924                 log_warn("accepted too many tcp, connections full");
 925                 return;
 926         }
 927         /* accept incoming connection. */
 928         c_hdl = c->tcp_free;
 929         /* clear leftover flags from previous use, and then set the
 930          * correct event base for the event structure for libevent */
 931         ub_event_free(c_hdl->ev->ev);
 932         c_hdl->ev->ev = ub_event_new(c_hdl->ev->base->eb->base, -1, UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT, comm_point_tcp_handle_callback, c_hdl);
 933         if(!c_hdl->ev->ev) {
 934                 log_warn("could not ub_event_new, dropped tcp");
 935                 return;
 936         }
 937         log_assert(fd != -1);
 938         (void)fd;
 939         new_fd = comm_point_perform_accept(c, &c_hdl->repinfo.addr,
 940                 &c_hdl->repinfo.addrlen);
 941         if(new_fd == -1)
 942                 return;
 943         if(c->ssl) {
 944                 c_hdl->ssl = incoming_ssl_fd(c->ssl, new_fd);
 945                 if(!c_hdl->ssl) {
 946                         c_hdl->fd = new_fd;
 947                         comm_point_close(c_hdl);
 948                         return;
 949                 }
 950                 c_hdl->ssl_shake_state = comm_ssl_shake_read;
 951 #ifdef USE_WINSOCK
 952                 comm_point_tcp_win_bio_cb(c_hdl, c_hdl->ssl);
 953 #endif
 954         }
 955
 956         /* grab the tcp handler buffers */
 957         c->cur_tcp_count++;
 958         c->tcp_free = c_hdl->tcp_free;
 959         if(!c->tcp_free) {
 960                 /* stop accepting incoming queries for now. */
 961                 comm_point_stop_listening(c);
 962         }
 963         setup_tcp_handler(c_hdl, new_fd, c->cur_tcp_count, c->max_tcp_count);
 964 }
 965
 966 /** Make tcp handler free for next assignment */
 967 static void
 968 reclaim_tcp_handler(struct comm_point* c)
 969 {
 970         log_assert(c->type == comm_tcp);
 971         if(c->ssl) {
 972 #ifdef HAVE_SSL
 973                 SSL_shutdown(c->ssl);
 974                 SSL_free(c->ssl);
 975                 c->ssl = NULL;
 976 #endif
 977         }
 978         comm_point_close(c);
 979         if(c->tcp_parent) {
 980                 c->tcp_parent->cur_tcp_count--;
 981                 c->tcp_free = c->tcp_parent->tcp_free;
 982                 c->tcp_parent->tcp_free = c;
 983                 if(!c->tcp_free) {
 984                         /* re-enable listening on accept socket */
 985                         comm_point_start_listening(c->tcp_parent, -1, -1);
 986                 }
 987         }
 988 }
 989
 990 /** do the callback when writing is done */
 991 static void
 992 tcp_callback_writer(struct comm_point* c)
 993 {
 994         log_assert(c->type == comm_tcp);
 995         sldns_buffer_clear(c->buffer);
 996         if(c->tcp_do_toggle_rw)
 997                 c->tcp_is_reading = 1;
 998         c->tcp_byte_count = 0;
 999         /* switch from listening(write) to listening(read) */
1000         if(c->tcp_req_info) {
1001                 tcp_req_info_handle_writedone(c->tcp_req_info);
1002         } else {
1003                 comm_point_stop_listening(c);
1004                 comm_point_start_listening(c, -1, c->tcp_timeout_msec);
1005         }
1006 }
1007
1008 /** do the callback when reading is done */
1009 static void
1010 tcp_callback_reader(struct comm_point* c)
1011 {
1012         log_assert(c->type == comm_tcp || c->type == comm_local);
1013         sldns_buffer_flip(c->buffer);
1014         if(c->tcp_do_toggle_rw)
1015                 c->tcp_is_reading = 0;
1016         c->tcp_byte_count = 0;
1017         if(c->tcp_req_info) {
1018                 tcp_req_info_handle_readdone(c->tcp_req_info);
1019         } else {
1020                 if(c->type == comm_tcp)
1021                         comm_point_stop_listening(c);
1022                 fptr_ok(fptr_whitelist_comm_point(c->callback));
1023                 if( (*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &c->repinfo) ) {
1024                         comm_point_start_listening(c, -1, c->tcp_timeout_msec);
1025                 }
1026         }
1027 }
1028
1029 #ifdef HAVE_SSL
1030 /** log certificate details */
1031 static void
1032 log_cert(unsigned level, const char* str, X509* cert)
1033 {
1034         BIO* bio;
1035         char nul = 0;
1036         char* pp = NULL;
1037         long len;
1038         if(verbosity < level) return;
1039         bio = BIO_new(BIO_s_mem());
1040         if(!bio) return;
1041         X509_print_ex(bio, cert, 0, (unsigned long)-1
1042                 ^(X509_FLAG_NO_SUBJECT
1043                         |X509_FLAG_NO_ISSUER|X509_FLAG_NO_VALIDITY
1044                         |X509_FLAG_NO_EXTENSIONS|X509_FLAG_NO_AUX
1045                         |X509_FLAG_NO_ATTRIBUTES));
1046         BIO_write(bio, &nul, (int)sizeof(nul));
1047         len = BIO_get_mem_data(bio, &pp);
1048         if(len != 0 && pp) {
1049                 verbose(level, "%s: \n%s", str, pp);
1050         }
1051         BIO_free(bio);
1052 }
1053 #endif /* HAVE_SSL */
1054
1055 #ifdef HAVE_SSL
1056 /** true if the ssl handshake error has to be squelched from the logs */
1057 static int
1058 squelch_err_ssl_handshake(unsigned long err)
1059 {
1060         if(verbosity >= VERB_QUERY)
1061                 return 0; /* only squelch on low verbosity */
1062         /* this is very specific, we could filter on ERR_GET_REASON()
1063          * (the third element in ERR_PACK) */
1064         if(err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_HTTPS_PROXY_REQUEST) ||
1065                 err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_HTTP_REQUEST) ||
1066                 err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_WRONG_VERSION_NUMBER) ||
1067                 err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_READ_BYTES, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE)
1068 #ifdef SSL_F_TLS_POST_PROCESS_CLIENT_HELLO
1069                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_POST_PROCESS_CLIENT_HELLO, SSL_R_NO_SHARED_CIPHER)
1070 #endif
1071 #ifdef SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO
1072                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_UNKNOWN_PROTOCOL)
1073                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_UNSUPPORTED_PROTOCOL)
1074 #  ifdef SSL_R_VERSION_TOO_LOW
1075                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_VERSION_TOO_LOW)
1076 #  endif
1077 #endif
1078                 )
1079                 return 1;
1080         return 0;
1081 }
1082 #endif /* HAVE_SSL */
1083
1084 /** continue ssl handshake */
1085 #ifdef HAVE_SSL
1086 static int
1087 ssl_handshake(struct comm_point* c)
1088 {
1089         int r;
1090         if(c->ssl_shake_state == comm_ssl_shake_hs_read) {
1091                 /* read condition satisfied back to writing */
1092                 comm_point_listen_for_rw(c, 1, 1);
1093                 c->ssl_shake_state = comm_ssl_shake_none;
1094                 return 1;
1095         }
1096         if(c->ssl_shake_state == comm_ssl_shake_hs_write) {
1097                 /* write condition satisfied, back to reading */
1098                 comm_point_listen_for_rw(c, 1, 0);
1099                 c->ssl_shake_state = comm_ssl_shake_none;
1100                 return 1;
1101         }
1102
1103         ERR_clear_error();
1104         r = SSL_do_handshake(c->ssl);
1105         if(r != 1) {
1106                 int want = SSL_get_error(c->ssl, r);
1107                 if(want == SSL_ERROR_WANT_READ) {
1108                         if(c->ssl_shake_state == comm_ssl_shake_read)
1109                                 return 1;
1110                         c->ssl_shake_state = comm_ssl_shake_read;
1111                         comm_point_listen_for_rw(c, 1, 0);
1112                         return 1;
1113                 } else if(want == SSL_ERROR_WANT_WRITE) {
1114                         if(c->ssl_shake_state == comm_ssl_shake_write)
1115                                 return 1;
1116                         c->ssl_shake_state = comm_ssl_shake_write;
1117                         comm_point_listen_for_rw(c, 0, 1);
1118                         return 1;
1119                 } else if(r == 0) {
1120                         return 0; /* closed */
1121                 } else if(want == SSL_ERROR_SYSCALL) {
1122                         /* SYSCALL and errno==0 means closed uncleanly */
1123 #ifdef EPIPE
1124                         if(errno == EPIPE && verbosity < 2)
1125                                 return 0; /* silence 'broken pipe' */
1126 #endif
1127 #ifdef ECONNRESET
1128                         if(errno == ECONNRESET && verbosity < 2)
1129                                 return 0; /* silence reset by peer */
1130 #endif
1131                         if(errno != 0)
1132                                 log_err("SSL_handshake syscall: %s",
1133                                         strerror(errno));
1134                         return 0;
1135                 } else {
1136                         unsigned long err = ERR_get_error();
1137                         if(!squelch_err_ssl_handshake(err)) {
1138                                 log_crypto_err_code("ssl handshake failed", err);
1139                                 log_addr(VERB_OPS, "ssl handshake failed", &c->repinfo.addr,
1140                                         c->repinfo.addrlen);
1141                         }
1142                         return 0;
1143                 }
1144         }
1145         /* this is where peer verification could take place */
1146         if((SSL_get_verify_mode(c->ssl)&SSL_VERIFY_PEER)) {
1147                 /* verification */
1148                 if(SSL_get_verify_result(c->ssl) == X509_V_OK) {
1149                         X509* x = SSL_get_peer_certificate(c->ssl);
1150                         if(!x) {
1151                                 log_addr(VERB_ALGO, "SSL connection failed: "
1152                                         "no certificate",
1153                                         &c->repinfo.addr, c->repinfo.addrlen);
1154                                 return 0;
1155                         }
1156                         log_cert(VERB_ALGO, "peer certificate", x);
1157 #ifdef HAVE_SSL_GET0_PEERNAME
1158                         if(SSL_get0_peername(c->ssl)) {
1159                                 char buf[255];
1160                                 snprintf(buf, sizeof(buf), "SSL connection "
1161                                         "to %s authenticated",
1162                                         SSL_get0_peername(c->ssl));
1163                                 log_addr(VERB_ALGO, buf, &c->repinfo.addr,
1164                                         c->repinfo.addrlen);
1165                         } else {
1166 #endif
1167                                 log_addr(VERB_ALGO, "SSL connection "
1168                                         "authenticated", &c->repinfo.addr,
1169                                         c->repinfo.addrlen);
1170 #ifdef HAVE_SSL_GET0_PEERNAME
1171                         }
1172 #endif
1173                         X509_free(x);
1174                 } else {
1175                         X509* x = SSL_get_peer_certificate(c->ssl);
1176                         if(x) {
1177                                 log_cert(VERB_ALGO, "peer certificate", x);
1178                                 X509_free(x);
1179                         }
1180                         log_addr(VERB_ALGO, "SSL connection failed: "
1181                                 "failed to authenticate",
1182                                 &c->repinfo.addr, c->repinfo.addrlen);
1183                         return 0;
1184                 }
1185         } else {
1186                 /* unauthenticated, the verify peer flag was not set
1187                  * in c->ssl when the ssl object was created from ssl_ctx */
1188                 log_addr(VERB_ALGO, "SSL connection", &c->repinfo.addr,
1189                         c->repinfo.addrlen);
1190         }
1191
1192         /* setup listen rw correctly */
1193         if(c->tcp_is_reading) {
1194                 if(c->ssl_shake_state != comm_ssl_shake_read)
1195                         comm_point_listen_for_rw(c, 1, 0);
1196         } else {
1197                 comm_point_listen_for_rw(c, 1, 1);
1198         }
1199         c->ssl_shake_state = comm_ssl_shake_none;
1200         return 1;
1201 }
1202 #endif /* HAVE_SSL */
1203
1204 /** ssl read callback on TCP */
1205 static int
1206 ssl_handle_read(struct comm_point* c)
1207 {
1208 #ifdef HAVE_SSL
1209         int r;
1210         if(c->ssl_shake_state != comm_ssl_shake_none) {
1211                 if(!ssl_handshake(c))
1212                         return 0;
1213                 if(c->ssl_shake_state != comm_ssl_shake_none)
1214                         return 1;
1215         }
1216         if(c->tcp_byte_count < sizeof(uint16_t)) {
1217                 /* read length bytes */
1218                 ERR_clear_error();
1219                 if((r=SSL_read(c->ssl, (void*)sldns_buffer_at(c->buffer,
1220                         c->tcp_byte_count), (int)(sizeof(uint16_t) -
1221                         c->tcp_byte_count))) <= 0) {
1222                         int want = SSL_get_error(c->ssl, r);
1223                         if(want == SSL_ERROR_ZERO_RETURN) {
1224                                 if(c->tcp_req_info)
1225                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1226                                 return 0; /* shutdown, closed */
1227                         } else if(want == SSL_ERROR_WANT_READ) {
1228                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1229                                 return 1; /* read more later */
1230                         } else if(want == SSL_ERROR_WANT_WRITE) {
1231                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1232                                 comm_point_listen_for_rw(c, 0, 1);
1233                                 return 1;
1234                         } else if(want == SSL_ERROR_SYSCALL) {
1235 #ifdef ECONNRESET
1236                                 if(errno == ECONNRESET && verbosity < 2)
1237                                         return 0; /* silence reset by peer */
1238 #endif
1239                                 if(errno != 0)
1240                                         log_err("SSL_read syscall: %s",
1241                                                 strerror(errno));
1242                                 return 0;
1243                         }
1244                         log_crypto_err("could not SSL_read");
1245                         return 0;
1246                 }
1247                 c->tcp_byte_count += r;
1248                 if(c->tcp_byte_count < sizeof(uint16_t))
1249                         return 1;
1250                 if(sldns_buffer_read_u16_at(c->buffer, 0) >
1251                         sldns_buffer_capacity(c->buffer)) {
1252                         verbose(VERB_QUERY, "ssl: dropped larger than buffer");
1253                         return 0;
1254                 }
1255                 sldns_buffer_set_limit(c->buffer,
1256                         sldns_buffer_read_u16_at(c->buffer, 0));
1257                 if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1258                         verbose(VERB_QUERY, "ssl: dropped bogus too short.");
1259                         return 0;
1260                 }
1261                 sldns_buffer_skip(c->buffer, (ssize_t)(c->tcp_byte_count-sizeof(uint16_t)));
1262                 verbose(VERB_ALGO, "Reading ssl tcp query of length %d",
1263                         (int)sldns_buffer_limit(c->buffer));
1264         }
1265         if(sldns_buffer_remaining(c->buffer) > 0) {
1266                 ERR_clear_error();
1267                 r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1268                         (int)sldns_buffer_remaining(c->buffer));
1269                 if(r <= 0) {
1270                         int want = SSL_get_error(c->ssl, r);
1271                         if(want == SSL_ERROR_ZERO_RETURN) {
1272                                 if(c->tcp_req_info)
1273                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1274                                 return 0; /* shutdown, closed */
1275                         } else if(want == SSL_ERROR_WANT_READ) {
1276                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1277                                 return 1; /* read more later */
1278                         } else if(want == SSL_ERROR_WANT_WRITE) {
1279                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1280                                 comm_point_listen_for_rw(c, 0, 1);
1281                                 return 1;
1282                         } else if(want == SSL_ERROR_SYSCALL) {
1283 #ifdef ECONNRESET
1284                                 if(errno == ECONNRESET && verbosity < 2)
1285                                         return 0; /* silence reset by peer */
1286 #endif
1287                                 if(errno != 0)
1288                                         log_err("SSL_read syscall: %s",
1289                                                 strerror(errno));
1290                                 return 0;
1291                         }
1292                         log_crypto_err("could not SSL_read");
1293                         return 0;
1294                 }
1295                 sldns_buffer_skip(c->buffer, (ssize_t)r);
1296         }
1297         if(sldns_buffer_remaining(c->buffer) <= 0) {
1298                 tcp_callback_reader(c);
1299         }
1300         return 1;
1301 #else
1302         (void)c;
1303         return 0;
1304 #endif /* HAVE_SSL */
1305 }
1306
1307 /** ssl write callback on TCP */
1308 static int
1309 ssl_handle_write(struct comm_point* c)
1310 {
1311 #ifdef HAVE_SSL
1312         int r;
1313         if(c->ssl_shake_state != comm_ssl_shake_none) {
1314                 if(!ssl_handshake(c))
1315                         return 0;
1316                 if(c->ssl_shake_state != comm_ssl_shake_none)
1317                         return 1;
1318         }
1319         /* ignore return, if fails we may simply block */
1320         (void)SSL_set_mode(c->ssl, (long)SSL_MODE_ENABLE_PARTIAL_WRITE);
1321         if(c->tcp_byte_count < sizeof(uint16_t)) {
1322                 uint16_t len = htons(sldns_buffer_limit(c->buffer));
1323                 ERR_clear_error();
1324                 if(sizeof(uint16_t)+sldns_buffer_remaining(c->buffer) <
1325                         LDNS_RR_BUF_SIZE) {
1326                         /* combine the tcp length and the query for write,
1327                          * this emulates writev */
1328                         uint8_t buf[LDNS_RR_BUF_SIZE];
1329                         memmove(buf, &len, sizeof(uint16_t));
1330                         memmove(buf+sizeof(uint16_t),
1331                                 sldns_buffer_current(c->buffer),
1332                                 sldns_buffer_remaining(c->buffer));
1333                         r = SSL_write(c->ssl, (void*)(buf+c->tcp_byte_count),
1334                                 (int)(sizeof(uint16_t)+
1335                                 sldns_buffer_remaining(c->buffer)
1336                                 - c->tcp_byte_count));
1337                 } else {
1338                         r = SSL_write(c->ssl,
1339                                 (void*)(((uint8_t*)&len)+c->tcp_byte_count),
1340                                 (int)(sizeof(uint16_t)-c->tcp_byte_count));
1341                 }
1342                 if(r <= 0) {
1343                         int want = SSL_get_error(c->ssl, r);
1344                         if(want == SSL_ERROR_ZERO_RETURN) {
1345                                 return 0; /* closed */
1346                         } else if(want == SSL_ERROR_WANT_READ) {
1347                                 c->ssl_shake_state = comm_ssl_shake_hs_read;
1348                                 comm_point_listen_for_rw(c, 1, 0);
1349                                 return 1; /* wait for read condition */
1350                         } else if(want == SSL_ERROR_WANT_WRITE) {
1351                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1352                                 return 1; /* write more later */
1353                         } else if(want == SSL_ERROR_SYSCALL) {
1354 #ifdef EPIPE
1355                                 if(errno == EPIPE && verbosity < 2)
1356                                         return 0; /* silence 'broken pipe' */
1357 #endif
1358                                 if(errno != 0)
1359                                         log_err("SSL_write syscall: %s",
1360                                                 strerror(errno));
1361                                 return 0;
1362                         }
1363                         log_crypto_err("could not SSL_write");
1364                         return 0;
1365                 }
1366                 c->tcp_byte_count += r;
1367                 if(c->tcp_byte_count < sizeof(uint16_t))
1368                         return 1;
1369                 sldns_buffer_set_position(c->buffer, c->tcp_byte_count -
1370                         sizeof(uint16_t));
1371                 if(sldns_buffer_remaining(c->buffer) == 0) {
1372                         tcp_callback_writer(c);
1373                         return 1;
1374                 }
1375         }
1376         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1377         ERR_clear_error();
1378         r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
1379                 (int)sldns_buffer_remaining(c->buffer));
1380         if(r <= 0) {
1381                 int want = SSL_get_error(c->ssl, r);
1382                 if(want == SSL_ERROR_ZERO_RETURN) {
1383                         return 0; /* closed */
1384                 } else if(want == SSL_ERROR_WANT_READ) {
1385                         c->ssl_shake_state = comm_ssl_shake_hs_read;
1386                         comm_point_listen_for_rw(c, 1, 0);
1387                         return 1; /* wait for read condition */
1388                 } else if(want == SSL_ERROR_WANT_WRITE) {
1389                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1390                         return 1; /* write more later */
1391                 } else if(want == SSL_ERROR_SYSCALL) {
1392 #ifdef EPIPE
1393                         if(errno == EPIPE && verbosity < 2)
1394                                 return 0; /* silence 'broken pipe' */
1395 #endif
1396                         if(errno != 0)
1397                                 log_err("SSL_write syscall: %s",
1398                                         strerror(errno));
1399                         return 0;
1400                 }
1401                 log_crypto_err("could not SSL_write");
1402                 return 0;
1403         }
1404         sldns_buffer_skip(c->buffer, (ssize_t)r);
1405
1406         if(sldns_buffer_remaining(c->buffer) == 0) {
1407                 tcp_callback_writer(c);
1408         }
1409         return 1;
1410 #else
1411         (void)c;
1412         return 0;
1413 #endif /* HAVE_SSL */
1414 }
1415
1416 /** handle ssl tcp connection with dns contents */
1417 static int
1418 ssl_handle_it(struct comm_point* c)
1419 {
1420         if(c->tcp_is_reading)
1421                 return ssl_handle_read(c);
1422         return ssl_handle_write(c);
1423 }
1424
1425 /** Handle tcp reading callback.
1426  * @param fd: file descriptor of socket.
1427  * @param c: comm point to read from into buffer.
1428  * @param short_ok: if true, very short packets are OK (for comm_local).
1429  * @return: 0 on error
1430  */
1431 static int
1432 comm_point_tcp_handle_read(int fd, struct comm_point* c, int short_ok)
1433 {
1434         ssize_t r;
1435         log_assert(c->type == comm_tcp || c->type == comm_local);
1436         if(c->ssl)
1437                 return ssl_handle_it(c);
1438         if(!c->tcp_is_reading)
1439                 return 0;
1440
1441         log_assert(fd != -1);
1442         if(c->tcp_byte_count < sizeof(uint16_t)) {
1443                 /* read length bytes */
1444                 r = recv(fd,(void*)sldns_buffer_at(c->buffer,c->tcp_byte_count),
1445                         sizeof(uint16_t)-c->tcp_byte_count, 0);
1446                 if(r == 0) {
1447                         if(c->tcp_req_info)
1448                                 return tcp_req_info_handle_read_close(c->tcp_req_info);
1449                         return 0;
1450                 } else if(r == -1) {
1451 #ifndef USE_WINSOCK
1452                         if(errno == EINTR || errno == EAGAIN)
1453                                 return 1;
1454 #ifdef ECONNRESET
1455                         if(errno == ECONNRESET && verbosity < 2)
1456                                 return 0; /* silence reset by peer */
1457 #endif
1458                         log_err_addr("read (in tcp s)", strerror(errno),
1459                                 &c->repinfo.addr, c->repinfo.addrlen);
1460 #else /* USE_WINSOCK */
1461                         if(WSAGetLastError() == WSAECONNRESET)
1462                                 return 0;
1463                         if(WSAGetLastError() == WSAEINPROGRESS)
1464                                 return 1;
1465                         if(WSAGetLastError() == WSAEWOULDBLOCK) {
1466                                 ub_winsock_tcp_wouldblock(c->ev->ev,
1467                                         UB_EV_READ);
1468                                 return 1;
1469                         }
1470                         log_err_addr("read (in tcp s)",
1471                                 wsa_strerror(WSAGetLastError()),
1472                                 &c->repinfo.addr, c->repinfo.addrlen);
1473 #endif
1474                         return 0;
1475                 }
1476                 c->tcp_byte_count += r;
1477                 if(c->tcp_byte_count != sizeof(uint16_t))
1478                         return 1;
1479                 if(sldns_buffer_read_u16_at(c->buffer, 0) >
1480                         sldns_buffer_capacity(c->buffer)) {
1481                         verbose(VERB_QUERY, "tcp: dropped larger than buffer");
1482                         return 0;
1483                 }
1484                 sldns_buffer_set_limit(c->buffer,
1485                         sldns_buffer_read_u16_at(c->buffer, 0));
1486                 if(!short_ok &&
1487                         sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1488                         verbose(VERB_QUERY, "tcp: dropped bogus too short.");
1489                         return 0;
1490                 }
1491                 verbose(VERB_ALGO, "Reading tcp query of length %d",
1492                         (int)sldns_buffer_limit(c->buffer));
1493         }
1494
1495         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1496         r = recv(fd, (void*)sldns_buffer_current(c->buffer),
1497                 sldns_buffer_remaining(c->buffer), 0);
1498         if(r == 0) {
1499                 if(c->tcp_req_info)
1500                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1501                 return 0;
1502         } else if(r == -1) {
1503 #ifndef USE_WINSOCK
1504                 if(errno == EINTR || errno == EAGAIN)
1505                         return 1;
1506                 log_err_addr("read (in tcp r)", strerror(errno),
1507                         &c->repinfo.addr, c->repinfo.addrlen);
1508 #else /* USE_WINSOCK */
1509                 if(WSAGetLastError() == WSAECONNRESET)
1510                         return 0;
1511                 if(WSAGetLastError() == WSAEINPROGRESS)
1512                         return 1;
1513                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1514                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1515                         return 1;
1516                 }
1517                 log_err_addr("read (in tcp r)",
1518                         wsa_strerror(WSAGetLastError()),
1519                         &c->repinfo.addr, c->repinfo.addrlen);
1520 #endif
1521                 return 0;
1522         }
1523         sldns_buffer_skip(c->buffer, r);
1524         if(sldns_buffer_remaining(c->buffer) <= 0) {
1525                 tcp_callback_reader(c);
1526         }
1527         return 1;
1528 }
1529
1530 /**
1531  * Handle tcp writing callback.
1532  * @param fd: file descriptor of socket.
1533  * @param c: comm point to write buffer out of.
1534  * @return: 0 on error
1535  */
1536 static int
1537 comm_point_tcp_handle_write(int fd, struct comm_point* c)
1538 {
1539         ssize_t r;
1540         struct sldns_buffer *buffer;
1541         log_assert(c->type == comm_tcp);
1542 #ifdef USE_DNSCRYPT
1543         buffer = c->dnscrypt_buffer;
1544 #else
1545         buffer = c->buffer;
1546 #endif
1547         if(c->tcp_is_reading && !c->ssl)
1548                 return 0;
1549         log_assert(fd != -1);
1550         if(c->tcp_byte_count == 0 && c->tcp_check_nb_connect) {
1551                 /* check for pending error from nonblocking connect */
1552                 /* from Stevens, unix network programming, vol1, 3rd ed, p450*/
1553                 int error = 0;
1554                 socklen_t len = (socklen_t)sizeof(error);
1555                 if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
1556                         &len) < 0){
1557 #ifndef USE_WINSOCK
1558                         error = errno; /* on solaris errno is error */
1559 #else /* USE_WINSOCK */
1560                         error = WSAGetLastError();
1561 #endif
1562                 }
1563 #ifndef USE_WINSOCK
1564 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
1565                 if(error == EINPROGRESS || error == EWOULDBLOCK)
1566                         return 1; /* try again later */
1567                 else
1568 #endif
1569                 if(error != 0 && verbosity < 2)
1570                         return 0; /* silence lots of chatter in the logs */
1571                 else if(error != 0) {
1572                         log_err_addr("tcp connect", strerror(error),
1573                                 &c->repinfo.addr, c->repinfo.addrlen);
1574 #else /* USE_WINSOCK */
1575                 /* examine error */
1576                 if(error == WSAEINPROGRESS)
1577                         return 1;
1578                 else if(error == WSAEWOULDBLOCK) {
1579                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1580                         return 1;
1581                 } else if(error != 0 && verbosity < 2)
1582                         return 0;
1583                 else if(error != 0) {
1584                         log_err_addr("tcp connect", wsa_strerror(error),
1585                                 &c->repinfo.addr, c->repinfo.addrlen);
1586 #endif /* USE_WINSOCK */
1587                         return 0;
1588                 }
1589         }
1590         if(c->ssl)
1591                 return ssl_handle_it(c);
1592
1593 #ifdef USE_MSG_FASTOPEN
1594         /* Only try this on first use of a connection that uses tfo,
1595            otherwise fall through to normal write */
1596         /* Also, TFO support on WINDOWS not implemented at the moment */
1597         if(c->tcp_do_fastopen == 1) {
1598                 /* this form of sendmsg() does both a connect() and send() so need to
1599                    look for various flavours of error*/
1600                 uint16_t len = htons(sldns_buffer_limit(buffer));
1601                 struct msghdr msg;
1602                 struct iovec iov[2];
1603                 c->tcp_do_fastopen = 0;
1604                 memset(&msg, 0, sizeof(msg));
1605                 iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
1606                 iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
1607                 iov[1].iov_base = sldns_buffer_begin(buffer);
1608                 iov[1].iov_len = sldns_buffer_limit(buffer);
1609                 log_assert(iov[0].iov_len > 0);
1610                 msg.msg_name = &c->repinfo.addr;
1611                 msg.msg_namelen = c->repinfo.addrlen;
1612                 msg.msg_iov = iov;
1613                 msg.msg_iovlen = 2;
1614                 r = sendmsg(fd, &msg, MSG_FASTOPEN);
1615                 if (r == -1) {
1616 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
1617                         /* Handshake is underway, maybe because no TFO cookie available.
1618                            Come back to write the message*/
1619                         if(errno == EINPROGRESS || errno == EWOULDBLOCK)
1620                                 return 1;
1621 #endif
1622                         if(errno == EINTR || errno == EAGAIN)
1623                                 return 1;
1624                         /* Not handling EISCONN here as shouldn't ever hit that case.*/
1625                         if(errno != EPIPE && errno != 0 && verbosity < 2)
1626                                 return 0; /* silence lots of chatter in the logs */
1627                         if(errno != EPIPE && errno != 0) {
1628                                 log_err_addr("tcp sendmsg", strerror(errno),
1629                                         &c->repinfo.addr, c->repinfo.addrlen);
1630                                 return 0;
1631                         }
1632                         /* fallthrough to nonFASTOPEN
1633                          * (MSG_FASTOPEN on Linux 3 produces EPIPE)
1634                          * we need to perform connect() */
1635                         if(connect(fd, (struct sockaddr *)&c->repinfo.addr, c->repinfo.addrlen) == -1) {
1636 #ifdef EINPROGRESS
1637                                 if(errno == EINPROGRESS)
1638                                         return 1; /* wait until connect done*/
1639 #endif
1640 #ifdef USE_WINSOCK
1641                                 if(WSAGetLastError() == WSAEINPROGRESS ||
1642                                         WSAGetLastError() == WSAEWOULDBLOCK)
1643                                         return 1; /* wait until connect done*/
1644 #endif
1645                                 if(tcp_connect_errno_needs_log(
1646                                         (struct sockaddr *)&c->repinfo.addr, c->repinfo.addrlen)) {
1647                                         log_err_addr("outgoing tcp: connect after EPIPE for fastopen",
1648                                                 strerror(errno), &c->repinfo.addr, c->repinfo.addrlen);
1649                                 }
1650                                 return 0;
1651                         }
1652
1653                 } else {
1654                         c->tcp_byte_count += r;
1655                         if(c->tcp_byte_count < sizeof(uint16_t))
1656                                 return 1;
1657                         sldns_buffer_set_position(buffer, c->tcp_byte_count -
1658                                 sizeof(uint16_t));
1659                         if(sldns_buffer_remaining(buffer) == 0) {
1660                                 tcp_callback_writer(c);
1661                                 return 1;
1662                         }
1663                 }
1664         }
1665 #endif /* USE_MSG_FASTOPEN */
1666
1667         if(c->tcp_byte_count < sizeof(uint16_t)) {
1668                 uint16_t len = htons(sldns_buffer_limit(buffer));
1669 #ifdef HAVE_WRITEV
1670                 struct iovec iov[2];
1671                 iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
1672                 iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
1673                 iov[1].iov_base = sldns_buffer_begin(buffer);
1674                 iov[1].iov_len = sldns_buffer_limit(buffer);
1675                 log_assert(iov[0].iov_len > 0);
1676                 r = writev(fd, iov, 2);
1677 #else /* HAVE_WRITEV */
1678                 r = send(fd, (void*)(((uint8_t*)&len)+c->tcp_byte_count),
1679                         sizeof(uint16_t)-c->tcp_byte_count, 0);
1680 #endif /* HAVE_WRITEV */
1681                 if(r == -1) {
1682 #ifndef USE_WINSOCK
1683 #  ifdef EPIPE
1684                         if(errno == EPIPE && verbosity < 2)
1685                                 return 0; /* silence 'broken pipe' */
1686   #endif
1687                         if(errno == EINTR || errno == EAGAIN)
1688                                 return 1;
1689 #ifdef ECONNRESET
1690                         if(errno == ECONNRESET && verbosity < 2)
1691                                 return 0; /* silence reset by peer */
1692 #endif
1693 #  ifdef HAVE_WRITEV
1694                         log_err_addr("tcp writev", strerror(errno),
1695                                 &c->repinfo.addr, c->repinfo.addrlen);
1696 #  else /* HAVE_WRITEV */
1697                         log_err_addr("tcp send s", strerror(errno),
1698                                 &c->repinfo.addr, c->repinfo.addrlen);
1699 #  endif /* HAVE_WRITEV */
1700 #else
1701                         if(WSAGetLastError() == WSAENOTCONN)
1702                                 return 1;
1703                         if(WSAGetLastError() == WSAEINPROGRESS)
1704                                 return 1;
1705                         if(WSAGetLastError() == WSAEWOULDBLOCK) {
1706                                 ub_winsock_tcp_wouldblock(c->ev->ev,
1707                                         UB_EV_WRITE);
1708                                 return 1;
1709                         }
1710                         if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
1711                                 return 0; /* silence reset by peer */
1712                         log_err_addr("tcp send s",
1713                                 wsa_strerror(WSAGetLastError()),
1714                                 &c->repinfo.addr, c->repinfo.addrlen);
1715 #endif
1716                         return 0;
1717                 }
1718                 c->tcp_byte_count += r;
1719                 if(c->tcp_byte_count < sizeof(uint16_t))
1720                         return 1;
1721                 sldns_buffer_set_position(buffer, c->tcp_byte_count -
1722                         sizeof(uint16_t));
1723                 if(sldns_buffer_remaining(buffer) == 0) {
1724                         tcp_callback_writer(c);
1725                         return 1;
1726                 }
1727         }
1728         log_assert(sldns_buffer_remaining(buffer) > 0);
1729         r = send(fd, (void*)sldns_buffer_current(buffer),
1730                 sldns_buffer_remaining(buffer), 0);
1731         if(r == -1) {
1732 #ifndef USE_WINSOCK
1733                 if(errno == EINTR || errno == EAGAIN)
1734                         return 1;
1735 #ifdef ECONNRESET
1736                 if(errno == ECONNRESET && verbosity < 2)
1737                         return 0; /* silence reset by peer */
1738 #endif
1739                 log_err_addr("tcp send r", strerror(errno),
1740                         &c->repinfo.addr, c->repinfo.addrlen);
1741 #else
1742                 if(WSAGetLastError() == WSAEINPROGRESS)
1743                         return 1;
1744                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1745                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1746                         return 1;
1747                 }
1748                 if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
1749                         return 0; /* silence reset by peer */
1750                 log_err_addr("tcp send r", wsa_strerror(WSAGetLastError()),
1751                         &c->repinfo.addr, c->repinfo.addrlen);
1752 #endif
1753                 return 0;
1754         }
1755         sldns_buffer_skip(buffer, r);
1756
1757         if(sldns_buffer_remaining(buffer) == 0) {
1758                 tcp_callback_writer(c);
1759         }
1760
1761         return 1;
1762 }
1763
1764 /** read again to drain buffers when there could be more to read */
1765 static void
1766 tcp_req_info_read_again(int fd, struct comm_point* c)
1767 {
1768         while(c->tcp_req_info->read_again) {
1769                 int r;
1770                 c->tcp_req_info->read_again = 0;
1771                 if(c->tcp_is_reading)
1772                         r = comm_point_tcp_handle_read(fd, c, 0);
1773                 else    r = comm_point_tcp_handle_write(fd, c);
1774                 if(!r) {
1775                         reclaim_tcp_handler(c);
1776                         if(!c->tcp_do_close) {
1777                                 fptr_ok(fptr_whitelist_comm_point(
1778                                         c->callback));
1779                                 (void)(*c->callback)(c, c->cb_arg,
1780                                         NETEVENT_CLOSED, NULL);
1781                         }
1782                         return;
1783                 }
1784         }
1785 }
1786
1787 void
1788 comm_point_tcp_handle_callback(int fd, short event, void* arg)
1789 {
1790         struct comm_point* c = (struct comm_point*)arg;
1791         log_assert(c->type == comm_tcp);
1792         ub_comm_base_now(c->ev->base);
1793
1794 #ifdef USE_DNSCRYPT
1795         /* Initialize if this is a dnscrypt socket */
1796         if(c->tcp_parent) {
1797                 c->dnscrypt = c->tcp_parent->dnscrypt;
1798         }
1799         if(c->dnscrypt && c->dnscrypt_buffer == c->buffer) {
1800                 c->dnscrypt_buffer = sldns_buffer_new(sldns_buffer_capacity(c->buffer));
1801                 if(!c->dnscrypt_buffer) {
1802                         log_err("Could not allocate dnscrypt buffer");
1803                         reclaim_tcp_handler(c);
1804                         if(!c->tcp_do_close) {
1805                                 fptr_ok(fptr_whitelist_comm_point(
1806                                         c->callback));
1807                                 (void)(*c->callback)(c, c->cb_arg,
1808                                         NETEVENT_CLOSED, NULL);
1809                         }
1810                         return;
1811                 }
1812         }
1813 #endif
1814
1815         if(event&UB_EV_TIMEOUT) {
1816                 verbose(VERB_QUERY, "tcp took too long, dropped");
1817                 reclaim_tcp_handler(c);
1818                 if(!c->tcp_do_close) {
1819                         fptr_ok(fptr_whitelist_comm_point(c->callback));
1820                         (void)(*c->callback)(c, c->cb_arg,
1821                                 NETEVENT_TIMEOUT, NULL);
1822                 }
1823                 return;
1824         }
1825         if(event&UB_EV_READ) {
1826                 int has_tcpq = (c->tcp_req_info != NULL);
1827                 if(!comm_point_tcp_handle_read(fd, c, 0)) {
1828                         reclaim_tcp_handler(c);
1829                         if(!c->tcp_do_close) {
1830                                 fptr_ok(fptr_whitelist_comm_point(
1831                                         c->callback));
1832                                 (void)(*c->callback)(c, c->cb_arg,
1833                                         NETEVENT_CLOSED, NULL);
1834                         }
1835                 }
1836                 if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again)
1837                         tcp_req_info_read_again(fd, c);
1838                 return;
1839         }
1840         if(event&UB_EV_WRITE) {
1841                 int has_tcpq = (c->tcp_req_info != NULL);
1842                 if(!comm_point_tcp_handle_write(fd, c)) {
1843                         reclaim_tcp_handler(c);
1844                         if(!c->tcp_do_close) {
1845                                 fptr_ok(fptr_whitelist_comm_point(
1846                                         c->callback));
1847                                 (void)(*c->callback)(c, c->cb_arg,
1848                                         NETEVENT_CLOSED, NULL);
1849                         }
1850                 }
1851                 if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again)
1852                         tcp_req_info_read_again(fd, c);
1853                 return;
1854         }
1855         log_err("Ignored event %d for tcphdl.", event);
1856 }
1857
1858 /** Make http handler free for next assignment */
1859 static void
1860 reclaim_http_handler(struct comm_point* c)
1861 {
1862         log_assert(c->type == comm_http);
1863         if(c->ssl) {
1864 #ifdef HAVE_SSL
1865                 SSL_shutdown(c->ssl);
1866                 SSL_free(c->ssl);
1867                 c->ssl = NULL;
1868 #endif
1869         }
1870         comm_point_close(c);
1871         if(c->tcp_parent) {
1872                 c->tcp_parent->cur_tcp_count--;
1873                 c->tcp_free = c->tcp_parent->tcp_free;
1874                 c->tcp_parent->tcp_free = c;
1875                 if(!c->tcp_free) {
1876                         /* re-enable listening on accept socket */
1877                         comm_point_start_listening(c->tcp_parent, -1, -1);
1878                 }
1879         }
1880 }
1881
1882 /** read more data for http (with ssl) */
1883 static int
1884 ssl_http_read_more(struct comm_point* c)
1885 {
1886 #ifdef HAVE_SSL
1887         int r;
1888         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1889         ERR_clear_error();
1890         r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1891                 (int)sldns_buffer_remaining(c->buffer));
1892         if(r <= 0) {
1893                 int want = SSL_get_error(c->ssl, r);
1894                 if(want == SSL_ERROR_ZERO_RETURN) {
1895                         return 0; /* shutdown, closed */
1896                 } else if(want == SSL_ERROR_WANT_READ) {
1897                         return 1; /* read more later */
1898                 } else if(want == SSL_ERROR_WANT_WRITE) {
1899                         c->ssl_shake_state = comm_ssl_shake_hs_write;
1900                         comm_point_listen_for_rw(c, 0, 1);
1901                         return 1;
1902                 } else if(want == SSL_ERROR_SYSCALL) {
1903 #ifdef ECONNRESET
1904                         if(errno == ECONNRESET && verbosity < 2)
1905                                 return 0; /* silence reset by peer */
1906 #endif
1907                         if(errno != 0)
1908                                 log_err("SSL_read syscall: %s",
1909                                         strerror(errno));
1910                         return 0;
1911                 }
1912                 log_crypto_err("could not SSL_read");
1913                 return 0;
1914         }
1915         sldns_buffer_skip(c->buffer, (ssize_t)r);
1916         return 1;
1917 #else
1918         (void)c;
1919         return 0;
1920 #endif /* HAVE_SSL */
1921 }
1922
1923 /** read more data for http */
1924 static int
1925 http_read_more(int fd, struct comm_point* c)
1926 {
1927         ssize_t r;
1928         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1929         r = recv(fd, (void*)sldns_buffer_current(c->buffer),
1930                 sldns_buffer_remaining(c->buffer), 0);
1931         if(r == 0) {
1932                 return 0;
1933         } else if(r == -1) {
1934 #ifndef USE_WINSOCK
1935                 if(errno == EINTR || errno == EAGAIN)
1936                         return 1;
1937                 log_err_addr("read (in http r)", strerror(errno),
1938                         &c->repinfo.addr, c->repinfo.addrlen);
1939 #else /* USE_WINSOCK */
1940                 if(WSAGetLastError() == WSAECONNRESET)
1941                         return 0;
1942                 if(WSAGetLastError() == WSAEINPROGRESS)
1943                         return 1;
1944                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1945                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1946                         return 1;
1947                 }
1948                 log_err_addr("read (in http r)",
1949                         wsa_strerror(WSAGetLastError()),
1950                         &c->repinfo.addr, c->repinfo.addrlen);
1951 #endif
1952                 return 0;
1953         }
1954         sldns_buffer_skip(c->buffer, r);
1955         return 1;
1956 }
1957
1958 /** return true if http header has been read (one line complete) */
1959 static int
1960 http_header_done(sldns_buffer* buf)
1961 {
1962         size_t i;
1963         for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
1964                 /* there was a \r before the \n, but we ignore that */
1965                 if((char)sldns_buffer_read_u8_at(buf, i) == '\n')
1966                         return 1;
1967         }
1968         return 0;
1969 }
1970
1971 /** return character string into buffer for header line, moves buffer
1972  * past that line and puts zero terminator into linefeed-newline */
1973 static char*
1974 http_header_line(sldns_buffer* buf)
1975 {
1976         char* result = (char*)sldns_buffer_current(buf);
1977         size_t i;
1978         for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
1979                 /* terminate the string on the \r */
1980                 if((char)sldns_buffer_read_u8_at(buf, i) == '\r')
1981                         sldns_buffer_write_u8_at(buf, i, 0);
1982                 /* terminate on the \n and skip past the it and done */
1983                 if((char)sldns_buffer_read_u8_at(buf, i) == '\n') {
1984                         sldns_buffer_write_u8_at(buf, i, 0);
1985                         sldns_buffer_set_position(buf, i+1);
1986                         return result;
1987                 }
1988         }
1989         return NULL;
1990 }
1991
1992 /** move unread buffer to start and clear rest for putting the rest into it */
1993 static void
1994 http_moveover_buffer(sldns_buffer* buf)
1995 {
1996         size_t pos = sldns_buffer_position(buf);
1997         size_t len = sldns_buffer_remaining(buf);
1998         sldns_buffer_clear(buf);
1999         memmove(sldns_buffer_begin(buf), sldns_buffer_at(buf, pos), len);
2000         sldns_buffer_set_position(buf, len);
2001 }
2002
2003 /** a http header is complete, process it */
2004 static int
2005 http_process_initial_header(struct comm_point* c)
2006 {
2007         char* line = http_header_line(c->buffer);
2008         if(!line) return 1;
2009         verbose(VERB_ALGO, "http header: %s", line);
2010         if(strncasecmp(line, "HTTP/1.1 ", 9) == 0) {
2011                 /* check returncode */
2012                 if(line[9] != '2') {
2013                         verbose(VERB_ALGO, "http bad status %s", line+9);
2014                         return 0;
2015                 }
2016         } else if(strncasecmp(line, "Content-Length: ", 16) == 0) {
2017                 if(!c->http_is_chunked)
2018                         c->tcp_byte_count = (size_t)atoi(line+16);
2019         } else if(strncasecmp(line, "Transfer-Encoding: chunked", 19+7) == 0) {
2020                 c->tcp_byte_count = 0;
2021                 c->http_is_chunked = 1;
2022         } else if(line[0] == 0) {
2023                 /* end of initial headers */
2024                 c->http_in_headers = 0;
2025                 if(c->http_is_chunked)
2026                         c->http_in_chunk_headers = 1;
2027                 /* remove header text from front of buffer
2028                  * the buffer is going to be used to return the data segment
2029                  * itself and we don't want the header to get returned
2030                  * prepended with it */
2031                 http_moveover_buffer(c->buffer);
2032                 sldns_buffer_flip(c->buffer);
2033                 return 1;
2034         }
2035         /* ignore other headers */
2036         return 1;
2037 }
2038
2039 /** a chunk header is complete, process it, return 0=fail, 1=continue next
2040  * header line, 2=done with chunked transfer*/
2041 static int
2042 http_process_chunk_header(struct comm_point* c)
2043 {
2044         char* line = http_header_line(c->buffer);
2045         if(!line) return 1;
2046         if(c->http_in_chunk_headers == 3) {
2047                 verbose(VERB_ALGO, "http chunk trailer: %s", line);
2048                 /* are we done ? */
2049                 if(line[0] == 0 && c->tcp_byte_count == 0) {
2050                         /* callback of http reader when NETEVENT_DONE,
2051                          * end of data, with no data in buffer */
2052                         sldns_buffer_set_position(c->buffer, 0);
2053                         sldns_buffer_set_limit(c->buffer, 0);
2054                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2055                         (void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2056                         /* return that we are done */
2057                         return 2;
2058                 }
2059                 if(line[0] == 0) {
2060                         /* continue with header of the next chunk */
2061                         c->http_in_chunk_headers = 1;
2062                         /* remove header text from front of buffer */
2063                         http_moveover_buffer(c->buffer);
2064                         sldns_buffer_flip(c->buffer);
2065                         return 1;
2066                 }
2067                 /* ignore further trail headers */
2068                 return 1;
2069         }
2070         verbose(VERB_ALGO, "http chunk header: %s", line);
2071         if(c->http_in_chunk_headers == 1) {
2072                 /* read chunked start line */
2073                 char* end = NULL;
2074                 c->tcp_byte_count = (size_t)strtol(line, &end, 16);
2075                 if(end == line)
2076                         return 0;
2077                 c->http_in_chunk_headers = 0;
2078                 /* remove header text from front of buffer */
2079                 http_moveover_buffer(c->buffer);
2080                 sldns_buffer_flip(c->buffer);
2081                 if(c->tcp_byte_count == 0) {
2082                         /* done with chunks, process chunk_trailer lines */
2083                         c->http_in_chunk_headers = 3;
2084                 }
2085                 return 1;
2086         }
2087         /* ignore other headers */
2088         return 1;
2089 }
2090
2091 /** handle nonchunked data segment */
2092 static int
2093 http_nonchunk_segment(struct comm_point* c)
2094 {
2095         /* c->buffer at position..limit has new data we read in.
2096          * the buffer itself is full of nonchunked data.
2097          * we are looking to read tcp_byte_count more data
2098          * and then the transfer is done. */
2099         size_t remainbufferlen;
2100         size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
2101         if(c->tcp_byte_count <= got_now) {
2102                 /* done, this is the last data fragment */
2103                 c->http_stored = 0;
2104                 sldns_buffer_set_position(c->buffer, 0);
2105                 fptr_ok(fptr_whitelist_comm_point(c->callback));
2106                 (void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2107                 return 1;
2108         }
2109         c->tcp_byte_count -= got_now;
2110         /* if we have the buffer space,
2111          * read more data collected into the buffer */
2112         remainbufferlen = sldns_buffer_capacity(c->buffer) -
2113                 sldns_buffer_limit(c->buffer);
2114         if(remainbufferlen >= c->tcp_byte_count ||
2115                 remainbufferlen >= 2048) {
2116                 size_t total = sldns_buffer_limit(c->buffer);
2117                 sldns_buffer_clear(c->buffer);
2118                 sldns_buffer_set_position(c->buffer, total);
2119                 c->http_stored = total;
2120                 /* return and wait to read more */
2121                 return 1;
2122         }
2123         /* call callback with this data amount, then
2124          * wait for more */
2125         c->http_stored = 0;
2126         sldns_buffer_set_position(c->buffer, 0);
2127         fptr_ok(fptr_whitelist_comm_point(c->callback));
2128         (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
2129         /* c->callback has to buffer_clear(c->buffer). */
2130         /* return and wait to read more */
2131         return 1;
2132 }
2133
2134 /** handle nonchunked data segment, return 0=fail, 1=wait, 2=process more */
2135 static int
2136 http_chunked_segment(struct comm_point* c)
2137 {
2138         /* the c->buffer has from position..limit new data we read. */
2139         /* the current chunk has length tcp_byte_count.
2140          * once we read that read more chunk headers.
2141          */
2142         size_t remainbufferlen;
2143         size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
2144         if(c->tcp_byte_count <= got_now) {
2145                 /* the chunk has completed (with perhaps some extra data
2146                  * from next chunk header and next chunk) */
2147                 /* save too much info into temp buffer */
2148                 size_t fraglen;
2149                 struct comm_reply repinfo;
2150                 c->http_stored = 0;
2151                 sldns_buffer_skip(c->buffer, (ssize_t)c->tcp_byte_count);
2152                 sldns_buffer_clear(c->http_temp);
2153                 sldns_buffer_write(c->http_temp,
2154                         sldns_buffer_current(c->buffer),
2155                         sldns_buffer_remaining(c->buffer));
2156                 sldns_buffer_flip(c->http_temp);
2157
2158                 /* callback with this fragment */
2159                 fraglen = sldns_buffer_position(c->buffer);
2160                 sldns_buffer_set_position(c->buffer, 0);
2161                 sldns_buffer_set_limit(c->buffer, fraglen);
2162                 repinfo = c->repinfo;
2163                 fptr_ok(fptr_whitelist_comm_point(c->callback));
2164                 (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &repinfo);
2165                 /* c->callback has to buffer_clear(). */
2166
2167                 /* is commpoint deleted? */
2168                 if(!repinfo.c) {
2169                         return 1;
2170                 }
2171                 /* copy waiting info */
2172                 sldns_buffer_clear(c->buffer);
2173                 sldns_buffer_write(c->buffer,
2174                         sldns_buffer_begin(c->http_temp),
2175                         sldns_buffer_remaining(c->http_temp));
2176                 sldns_buffer_flip(c->buffer);
2177                 /* process end of chunk trailer header lines, until
2178                  * an empty line */
2179                 c->http_in_chunk_headers = 3;
2180                 /* process more data in buffer (if any) */
2181                 return 2;
2182         }
2183         c->tcp_byte_count -= got_now;
2184
2185         /* if we have the buffer space,
2186          * read more data collected into the buffer */
2187         remainbufferlen = sldns_buffer_capacity(c->buffer) -
2188                 sldns_buffer_limit(c->buffer);
2189         if(remainbufferlen >= c->tcp_byte_count ||
2190                 remainbufferlen >= 2048) {
2191                 size_t total = sldns_buffer_limit(c->buffer);
2192                 sldns_buffer_clear(c->buffer);
2193                 sldns_buffer_set_position(c->buffer, total);
2194                 c->http_stored = total;
2195                 /* return and wait to read more */
2196                 return 1;
2197         }
2198
2199         /* callback of http reader for a new part of the data */
2200         c->http_stored = 0;
2201         sldns_buffer_set_position(c->buffer, 0);
2202         fptr_ok(fptr_whitelist_comm_point(c->callback));
2203         (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
2204         /* c->callback has to buffer_clear(c->buffer). */
2205         /* return and wait to read more */
2206         return 1;
2207 }
2208
2209 /**
2210  * Handle http reading callback.
2211  * @param fd: file descriptor of socket.
2212  * @param c: comm point to read from into buffer.
2213  * @return: 0 on error
2214  */
2215 static int
2216 comm_point_http_handle_read(int fd, struct comm_point* c)
2217 {
2218         log_assert(c->type == comm_http);
2219         log_assert(fd != -1);
2220
2221         /* if we are in ssl handshake, handle SSL handshake */
2222 #ifdef HAVE_SSL
2223         if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
2224                 if(!ssl_handshake(c))
2225                         return 0;
2226                 if(c->ssl_shake_state != comm_ssl_shake_none)
2227                         return 1;
2228         }
2229 #endif /* HAVE_SSL */
2230
2231         if(!c->tcp_is_reading)
2232                 return 1;
2233         /* read more data */
2234         if(c->ssl) {
2235                 if(!ssl_http_read_more(c))
2236                         return 0;
2237         } else {
2238                 if(!http_read_more(fd, c))
2239                         return 0;
2240         }
2241
2242         sldns_buffer_flip(c->buffer);
2243         while(sldns_buffer_remaining(c->buffer) > 0) {
2244                 /* if we are reading headers, read more headers */
2245                 if(c->http_in_headers || c->http_in_chunk_headers) {
2246                         /* if header is done, process the header */
2247                         if(!http_header_done(c->buffer)) {
2248                                 /* copy remaining data to front of buffer
2249                                  * and set rest for writing into it */
2250                                 http_moveover_buffer(c->buffer);
2251                                 /* return and wait to read more */
2252                                 return 1;
2253                         }
2254                         if(!c->http_in_chunk_headers) {
2255                                 /* process initial headers */
2256                                 if(!http_process_initial_header(c))
2257                                         return 0;
2258                         } else {
2259                                 /* process chunk headers */
2260                                 int r = http_process_chunk_header(c);
2261                                 if(r == 0) return 0;
2262                                 if(r == 2) return 1; /* done */
2263                                 /* r == 1, continue */
2264                         }
2265                         /* see if we have more to process */
2266                         continue;
2267                 }
2268
2269                 if(!c->http_is_chunked) {
2270                         /* if we are reading nonchunks, process that*/
2271                         return http_nonchunk_segment(c);
2272                 } else {
2273                         /* if we are reading chunks, read the chunk */
2274                         int r = http_chunked_segment(c);
2275                         if(r == 0) return 0;
2276                         if(r == 1) return 1;
2277                         continue;
2278                 }
2279         }
2280         /* broke out of the loop; could not process header instead need
2281          * to read more */
2282         /* moveover any remaining data and read more data */
2283         http_moveover_buffer(c->buffer);
2284         /* return and wait to read more */
2285         return 1;
2286 }
2287
2288 /** check pending connect for http */
2289 static int
2290 http_check_connect(int fd, struct comm_point* c)
2291 {
2292         /* check for pending error from nonblocking connect */
2293         /* from Stevens, unix network programming, vol1, 3rd ed, p450*/
2294         int error = 0;
2295         socklen_t len = (socklen_t)sizeof(error);
2296         if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
2297                 &len) < 0){
2298 #ifndef USE_WINSOCK
2299                 error = errno; /* on solaris errno is error */
2300 #else /* USE_WINSOCK */
2301                 error = WSAGetLastError();
2302 #endif
2303         }
2304 #ifndef USE_WINSOCK
2305 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
2306         if(error == EINPROGRESS || error == EWOULDBLOCK)
2307                 return 1; /* try again later */
2308         else
2309 #endif
2310         if(error != 0 && verbosity < 2)
2311                 return 0; /* silence lots of chatter in the logs */
2312         else if(error != 0) {
2313                 log_err_addr("http connect", strerror(error),
2314                         &c->repinfo.addr, c->repinfo.addrlen);
2315 #else /* USE_WINSOCK */
2316         /* examine error */
2317         if(error == WSAEINPROGRESS)
2318                 return 1;
2319         else if(error == WSAEWOULDBLOCK) {
2320                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2321                 return 1;
2322         } else if(error != 0 && verbosity < 2)
2323                 return 0;
2324         else if(error != 0) {
2325                 log_err_addr("http connect", wsa_strerror(error),
2326                         &c->repinfo.addr, c->repinfo.addrlen);
2327 #endif /* USE_WINSOCK */
2328                 return 0;
2329         }
2330         /* keep on processing this socket */
2331         return 2;
2332 }
2333
2334 /** write more data for http (with ssl) */
2335 static int
2336 ssl_http_write_more(struct comm_point* c)
2337 {
2338 #ifdef HAVE_SSL
2339         int r;
2340         log_assert(sldns_buffer_remaining(c->buffer) > 0);
2341         ERR_clear_error();
2342         r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
2343                 (int)sldns_buffer_remaining(c->buffer));
2344         if(r <= 0) {
2345                 int want = SSL_get_error(c->ssl, r);
2346                 if(want == SSL_ERROR_ZERO_RETURN) {
2347                         return 0; /* closed */
2348                 } else if(want == SSL_ERROR_WANT_READ) {
2349                         c->ssl_shake_state = comm_ssl_shake_hs_read;
2350                         comm_point_listen_for_rw(c, 1, 0);
2351                         return 1; /* wait for read condition */
2352                 } else if(want == SSL_ERROR_WANT_WRITE) {
2353                         return 1; /* write more later */
2354                 } else if(want == SSL_ERROR_SYSCALL) {
2355 #ifdef EPIPE
2356                         if(errno == EPIPE && verbosity < 2)
2357                                 return 0; /* silence 'broken pipe' */
2358 #endif
2359                         if(errno != 0)
2360                                 log_err("SSL_write syscall: %s",
2361                                         strerror(errno));
2362                         return 0;
2363                 }
2364                 log_crypto_err("could not SSL_write");
2365                 return 0;
2366         }
2367         sldns_buffer_skip(c->buffer, (ssize_t)r);
2368         return 1;
2369 #else
2370         (void)c;
2371         return 0;
2372 #endif /* HAVE_SSL */
2373 }
2374
2375 /** write more data for http */
2376 static int
2377 http_write_more(int fd, struct comm_point* c)
2378 {
2379         ssize_t r;
2380         log_assert(sldns_buffer_remaining(c->buffer) > 0);
2381         r = send(fd, (void*)sldns_buffer_current(c->buffer),
2382                 sldns_buffer_remaining(c->buffer), 0);
2383         if(r == -1) {
2384 #ifndef USE_WINSOCK
2385                 if(errno == EINTR || errno == EAGAIN)
2386                         return 1;
2387                 log_err_addr("http send r", strerror(errno),
2388                         &c->repinfo.addr, c->repinfo.addrlen);
2389 #else
2390                 if(WSAGetLastError() == WSAEINPROGRESS)
2391                         return 1;
2392                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
2393                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2394                         return 1;
2395                 }
2396                 log_err_addr("http send r", wsa_strerror(WSAGetLastError()),
2397                         &c->repinfo.addr, c->repinfo.addrlen);
2398 #endif
2399                 return 0;
2400         }
2401         sldns_buffer_skip(c->buffer, r);
2402         return 1;
2403 }
2404
2405 /**
2406  * Handle http writing callback.
2407  * @param fd: file descriptor of socket.
2408  * @param c: comm point to write buffer out of.
2409  * @return: 0 on error
2410  */
2411 static int
2412 comm_point_http_handle_write(int fd, struct comm_point* c)
2413 {
2414         log_assert(c->type == comm_http);
2415         log_assert(fd != -1);
2416
2417         /* check pending connect errors, if that fails, we wait for more,
2418          * or we can continue to write contents */
2419         if(c->tcp_check_nb_connect) {
2420                 int r = http_check_connect(fd, c);
2421                 if(r == 0) return 0;
2422                 if(r == 1) return 1;
2423                 c->tcp_check_nb_connect = 0;
2424         }
2425         /* if we are in ssl handshake, handle SSL handshake */
2426 #ifdef HAVE_SSL
2427         if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
2428                 if(!ssl_handshake(c))
2429                         return 0;
2430                 if(c->ssl_shake_state != comm_ssl_shake_none)
2431                         return 1;
2432         }
2433 #endif /* HAVE_SSL */
2434         if(c->tcp_is_reading)
2435                 return 1;
2436         /* if we are writing, write more */
2437         if(c->ssl) {
2438                 if(!ssl_http_write_more(c))
2439                         return 0;
2440         } else {
2441                 if(!http_write_more(fd, c))
2442                         return 0;
2443         }
2444
2445         /* we write a single buffer contents, that can contain
2446          * the http request, and then flip to read the results */
2447         /* see if write is done */
2448         if(sldns_buffer_remaining(c->buffer) == 0) {
2449                 sldns_buffer_clear(c->buffer);
2450                 if(c->tcp_do_toggle_rw)
2451                         c->tcp_is_reading = 1;
2452                 c->tcp_byte_count = 0;
2453                 /* switch from listening(write) to listening(read) */
2454                 comm_point_stop_listening(c);
2455                 comm_point_start_listening(c, -1, -1);
2456         }
2457         return 1;
2458 }
2459
2460 void
2461 comm_point_http_handle_callback(int fd, short event, void* arg)
2462 {
2463         struct comm_point* c = (struct comm_point*)arg;
2464         log_assert(c->type == comm_http);
2465         ub_comm_base_now(c->ev->base);
2466
2467         if(event&UB_EV_TIMEOUT) {
2468                 verbose(VERB_QUERY, "http took too long, dropped");
2469                 reclaim_http_handler(c);
2470                 if(!c->tcp_do_close) {
2471                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2472                         (void)(*c->callback)(c, c->cb_arg,
2473                                 NETEVENT_TIMEOUT, NULL);
2474                 }
2475                 return;
2476         }
2477         if(event&UB_EV_READ) {
2478                 if(!comm_point_http_handle_read(fd, c)) {
2479                         reclaim_http_handler(c);
2480                         if(!c->tcp_do_close) {
2481                                 fptr_ok(fptr_whitelist_comm_point(
2482                                         c->callback));
2483                                 (void)(*c->callback)(c, c->cb_arg,
2484                                         NETEVENT_CLOSED, NULL);
2485                         }
2486                 }
2487                 return;
2488         }
2489         if(event&UB_EV_WRITE) {
2490                 if(!comm_point_http_handle_write(fd, c)) {
2491                         reclaim_http_handler(c);
2492                         if(!c->tcp_do_close) {
2493                                 fptr_ok(fptr_whitelist_comm_point(
2494                                         c->callback));
2495                                 (void)(*c->callback)(c, c->cb_arg,
2496                                         NETEVENT_CLOSED, NULL);
2497                         }
2498                 }
2499                 return;
2500         }
2501         log_err("Ignored event %d for httphdl.", event);
2502 }
2503
2504 void comm_point_local_handle_callback(int fd, short event, void* arg)
2505 {
2506         struct comm_point* c = (struct comm_point*)arg;
2507         log_assert(c->type == comm_local);
2508         ub_comm_base_now(c->ev->base);
2509
2510         if(event&UB_EV_READ) {
2511                 if(!comm_point_tcp_handle_read(fd, c, 1)) {
2512                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2513                         (void)(*c->callback)(c, c->cb_arg, NETEVENT_CLOSED,
2514                                 NULL);
2515                 }
2516                 return;
2517         }
2518         log_err("Ignored event %d for localhdl.", event);
2519 }
2520
2521 void comm_point_raw_handle_callback(int ATTR_UNUSED(fd),
2522         short event, void* arg)
2523 {
2524         struct comm_point* c = (struct comm_point*)arg;
2525         int err = NETEVENT_NOERROR;
2526         log_assert(c->type == comm_raw);
2527         ub_comm_base_now(c->ev->base);
2528
2529         if(event&UB_EV_TIMEOUT)
2530                 err = NETEVENT_TIMEOUT;
2531         fptr_ok(fptr_whitelist_comm_point_raw(c->callback));
2532         (void)(*c->callback)(c, c->cb_arg, err, NULL);
2533 }
2534
2535 struct comm_point*
2536 comm_point_create_udp(struct comm_base *base, int fd, sldns_buffer* buffer,
2537         comm_point_callback_type* callback, void* callback_arg)
2538 {
2539         struct comm_point* c = (struct comm_point*)calloc(1,
2540                 sizeof(struct comm_point));
2541         short evbits;
2542         if(!c)
2543                 return NULL;
2544         c->ev = (struct internal_event*)calloc(1,
2545                 sizeof(struct internal_event));
2546         if(!c->ev) {
2547                 free(c);
2548                 return NULL;
2549         }
2550         c->ev->base = base;
2551         c->fd = fd;
2552         c->buffer = buffer;
2553         c->timeout = NULL;
2554         c->tcp_is_reading = 0;
2555         c->tcp_byte_count = 0;
2556         c->tcp_parent = NULL;
2557         c->max_tcp_count = 0;
2558         c->cur_tcp_count = 0;
2559         c->tcp_handlers = NULL;
2560         c->tcp_free = NULL;
2561         c->type = comm_udp;
2562         c->tcp_do_close = 0;
2563         c->do_not_close = 0;
2564         c->tcp_do_toggle_rw = 0;
2565         c->tcp_check_nb_connect = 0;
2566 #ifdef USE_MSG_FASTOPEN
2567         c->tcp_do_fastopen = 0;
2568 #endif
2569 #ifdef USE_DNSCRYPT
2570         c->dnscrypt = 0;
2571         c->dnscrypt_buffer = buffer;
2572 #endif
2573         c->inuse = 0;
2574         c->callback = callback;
2575         c->cb_arg = callback_arg;
2576         evbits = UB_EV_READ | UB_EV_PERSIST;
2577         /* ub_event stuff */
2578         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2579                 comm_point_udp_callback, c);
2580         if(c->ev->ev == NULL) {
2581                 log_err("could not baseset udp event");
2582                 comm_point_delete(c);
2583                 return NULL;
2584         }
2585         if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
2586                 log_err("could not add udp event");
2587                 comm_point_delete(c);
2588                 return NULL;
2589         }
2590         return c;
2591 }
2592
2593 struct comm_point*
2594 comm_point_create_udp_ancil(struct comm_base *base, int fd,
2595         sldns_buffer* buffer,
2596         comm_point_callback_type* callback, void* callback_arg)
2597 {
2598         struct comm_point* c = (struct comm_point*)calloc(1,
2599                 sizeof(struct comm_point));
2600         short evbits;
2601         if(!c)
2602                 return NULL;
2603         c->ev = (struct internal_event*)calloc(1,
2604                 sizeof(struct internal_event));
2605         if(!c->ev) {
2606                 free(c);
2607                 return NULL;
2608         }
2609         c->ev->base = base;
2610         c->fd = fd;
2611         c->buffer = buffer;
2612         c->timeout = NULL;
2613         c->tcp_is_reading = 0;
2614         c->tcp_byte_count = 0;
2615         c->tcp_parent = NULL;
2616         c->max_tcp_count = 0;
2617         c->cur_tcp_count = 0;
2618         c->tcp_handlers = NULL;
2619         c->tcp_free = NULL;
2620         c->type = comm_udp;
2621         c->tcp_do_close = 0;
2622         c->do_not_close = 0;
2623 #ifdef USE_DNSCRYPT
2624         c->dnscrypt = 0;
2625         c->dnscrypt_buffer = buffer;
2626 #endif
2627         c->inuse = 0;
2628         c->tcp_do_toggle_rw = 0;
2629         c->tcp_check_nb_connect = 0;
2630 #ifdef USE_MSG_FASTOPEN
2631         c->tcp_do_fastopen = 0;
2632 #endif
2633         c->callback = callback;
2634         c->cb_arg = callback_arg;
2635         evbits = UB_EV_READ | UB_EV_PERSIST;
2636         /* ub_event stuff */
2637         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2638                 comm_point_udp_ancil_callback, c);
2639         if(c->ev->ev == NULL) {
2640                 log_err("could not baseset udp event");
2641                 comm_point_delete(c);
2642                 return NULL;
2643         }
2644         if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
2645                 log_err("could not add udp event");
2646                 comm_point_delete(c);
2647                 return NULL;
2648         }
2649         return c;
2650 }
2651
2652 static struct comm_point*
2653 comm_point_create_tcp_handler(struct comm_base *base,
2654         struct comm_point* parent, size_t bufsize,
2655         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
2656         void* callback_arg)
2657 {
2658         struct comm_point* c = (struct comm_point*)calloc(1,
2659                 sizeof(struct comm_point));
2660         short evbits;
2661         if(!c)
2662                 return NULL;
2663         c->ev = (struct internal_event*)calloc(1,
2664                 sizeof(struct internal_event));
2665         if(!c->ev) {
2666                 free(c);
2667                 return NULL;
2668         }
2669         c->ev->base = base;
2670         c->fd = -1;
2671         c->buffer = sldns_buffer_new(bufsize);
2672         if(!c->buffer) {
2673                 free(c->ev);
2674                 free(c);
2675                 return NULL;
2676         }
2677         c->timeout = (struct timeval*)malloc(sizeof(struct timeval));
2678         if(!c->timeout) {
2679                 sldns_buffer_free(c->buffer);
2680                 free(c->ev);
2681                 free(c);
2682                 return NULL;
2683         }
2684         c->tcp_is_reading = 0;
2685         c->tcp_byte_count = 0;
2686         c->tcp_parent = parent;
2687         c->tcp_timeout_msec = parent->tcp_timeout_msec;
2688         c->tcp_conn_limit = parent->tcp_conn_limit;
2689         c->tcl_addr = NULL;
2690         c->tcp_keepalive = 0;
2691         c->max_tcp_count = 0;
2692         c->cur_tcp_count = 0;
2693         c->tcp_handlers = NULL;
2694         c->tcp_free = NULL;
2695         c->type = comm_tcp;
2696         c->tcp_do_close = 0;
2697         c->do_not_close = 0;
2698         c->tcp_do_toggle_rw = 1;
2699         c->tcp_check_nb_connect = 0;
2700 #ifdef USE_MSG_FASTOPEN
2701         c->tcp_do_fastopen = 0;
2702 #endif
2703 #ifdef USE_DNSCRYPT
2704         c->dnscrypt = 0;
2705         /* We don't know just yet if this is a dnscrypt channel. Allocation
2706          * will be done when handling the callback. */
2707         c->dnscrypt_buffer = c->buffer;
2708 #endif
2709         c->repinfo.c = c;
2710         c->callback = callback;
2711         c->cb_arg = callback_arg;
2712         if(spoolbuf) {
2713                 c->tcp_req_info = tcp_req_info_create(spoolbuf);
2714                 if(!c->tcp_req_info) {
2715                         log_err("could not create tcp commpoint");
2716                         sldns_buffer_free(c->buffer);
2717                         free(c->timeout);
2718                         free(c->ev);
2719                         free(c);
2720                         return NULL;
2721                 }
2722                 c->tcp_req_info->cp = c;
2723                 c->tcp_do_close = 1;
2724                 c->tcp_do_toggle_rw = 0;
2725         }
2726         /* add to parent free list */
2727         c->tcp_free = parent->tcp_free;
2728         parent->tcp_free = c;
2729         /* ub_event stuff */
2730         evbits = UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT;
2731         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2732                 comm_point_tcp_handle_callback, c);
2733         if(c->ev->ev == NULL)
2734         {
2735                 log_err("could not basetset tcphdl event");
2736                 parent->tcp_free = c->tcp_free;
2737                 tcp_req_info_delete(c->tcp_req_info);
2738                 sldns_buffer_free(c->buffer);
2739                 free(c->timeout);
2740                 free(c->ev);
2741                 free(c);
2742                 return NULL;
2743         }
2744         return c;
2745 }
2746
2747 struct comm_point*
2748 comm_point_create_tcp(struct comm_base *base, int fd, int num,
2749         int idle_timeout, struct tcl_list* tcp_conn_limit, size_t bufsize,
2750         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
2751         void* callback_arg)
2752 {
2753         struct comm_point* c = (struct comm_point*)calloc(1,
2754                 sizeof(struct comm_point));
2755         short evbits;
2756         int i;
2757         /* first allocate the TCP accept listener */
2758         if(!c)
2759                 return NULL;
2760         c->ev = (struct internal_event*)calloc(1,
2761                 sizeof(struct internal_event));
2762         if(!c->ev) {
2763                 free(c);
2764                 return NULL;
2765         }
2766         c->ev->base = base;
2767         c->fd = fd;
2768         c->buffer = NULL;
2769         c->timeout = NULL;
2770         c->tcp_is_reading = 0;
2771         c->tcp_byte_count = 0;
2772         c->tcp_timeout_msec = idle_timeout;
2773         c->tcp_conn_limit = tcp_conn_limit;
2774         c->tcl_addr = NULL;
2775         c->tcp_keepalive = 0;
2776         c->tcp_parent = NULL;
2777         c->max_tcp_count = num;
2778         c->cur_tcp_count = 0;
2779         c->tcp_handlers = (struct comm_point**)calloc((size_t)num,
2780                 sizeof(struct comm_point*));
2781         if(!c->tcp_handlers) {
2782                 free(c->ev);
2783                 free(c);
2784                 return NULL;
2785         }
2786         c->tcp_free = NULL;
2787         c->type = comm_tcp_accept;
2788         c->tcp_do_close = 0;
2789         c->do_not_close = 0;
2790         c->tcp_do_toggle_rw = 0;
2791         c->tcp_check_nb_connect = 0;
2792 #ifdef USE_MSG_FASTOPEN
2793         c->tcp_do_fastopen = 0;
2794 #endif
2795 #ifdef USE_DNSCRYPT
2796         c->dnscrypt = 0;
2797         c->dnscrypt_buffer = NULL;
2798 #endif
2799         c->callback = NULL;
2800         c->cb_arg = NULL;
2801         evbits = UB_EV_READ | UB_EV_PERSIST;
2802         /* ub_event stuff */
2803         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2804                 comm_point_tcp_accept_callback, c);
2805         if(c->ev->ev == NULL) {
2806                 log_err("could not baseset tcpacc event");
2807                 comm_point_delete(c);
2808                 return NULL;
2809         }
2810         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
2811                 log_err("could not add tcpacc event");
2812                 comm_point_delete(c);
2813                 return NULL;
2814         }
2815         /* now prealloc the tcp handlers */
2816         for(i=0; i<num; i++) {
2817                 c->tcp_handlers[i] = comm_point_create_tcp_handler(base,
2818                         c, bufsize, spoolbuf, callback, callback_arg);
2819                 if(!c->tcp_handlers[i]) {
2820                         comm_point_delete(c);
2821                         return NULL;
2822                 }
2823         }
2824
2825         return c;
2826 }
2827
2828 struct comm_point*
2829 comm_point_create_tcp_out(struct comm_base *base, size_t bufsize,
2830         comm_point_callback_type* callback, void* callback_arg)
2831 {
2832         struct comm_point* c = (struct comm_point*)calloc(1,
2833                 sizeof(struct comm_point));
2834         short evbits;
2835         if(!c)
2836                 return NULL;
2837         c->ev = (struct internal_event*)calloc(1,
2838                 sizeof(struct internal_event));
2839         if(!c->ev) {
2840                 free(c);
2841                 return NULL;
2842         }
2843         c->ev->base = base;
2844         c->fd = -1;
2845         c->buffer = sldns_buffer_new(bufsize);
2846         if(!c->buffer) {
2847                 free(c->ev);
2848                 free(c);
2849                 return NULL;
2850         }
2851         c->timeout = NULL;
2852         c->tcp_is_reading = 0;
2853         c->tcp_byte_count = 0;
2854         c->tcp_timeout_msec = TCP_QUERY_TIMEOUT;
2855         c->tcp_conn_limit = NULL;
2856         c->tcl_addr = NULL;
2857         c->tcp_keepalive = 0;
2858         c->tcp_parent = NULL;
2859         c->max_tcp_count = 0;
2860         c->cur_tcp_count = 0;
2861         c->tcp_handlers = NULL;
2862         c->tcp_free = NULL;
2863         c->type = comm_tcp;
2864         c->tcp_do_close = 0;
2865         c->do_not_close = 0;
2866         c->tcp_do_toggle_rw = 1;
2867         c->tcp_check_nb_connect = 1;
2868 #ifdef USE_MSG_FASTOPEN
2869         c->tcp_do_fastopen = 1;
2870 #endif
2871 #ifdef USE_DNSCRYPT
2872         c->dnscrypt = 0;
2873         c->dnscrypt_buffer = c->buffer;
2874 #endif
2875         c->repinfo.c = c;
2876         c->callback = callback;
2877         c->cb_arg = callback_arg;
2878         evbits = UB_EV_PERSIST | UB_EV_WRITE;
2879         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2880                 comm_point_tcp_handle_callback, c);
2881         if(c->ev->ev == NULL)
2882         {
2883                 log_err("could not baseset tcpout event");
2884                 sldns_buffer_free(c->buffer);
2885                 free(c->ev);
2886                 free(c);
2887                 return NULL;
2888         }
2889
2890         return c;
2891 }
2892
2893 struct comm_point*
2894 comm_point_create_http_out(struct comm_base *base, size_t bufsize,
2895         comm_point_callback_type* callback, void* callback_arg,
2896         sldns_buffer* temp)
2897 {
2898         struct comm_point* c = (struct comm_point*)calloc(1,
2899                 sizeof(struct comm_point));
2900         short evbits;
2901         if(!c)
2902                 return NULL;
2903         c->ev = (struct internal_event*)calloc(1,
2904                 sizeof(struct internal_event));
2905         if(!c->ev) {
2906                 free(c);
2907                 return NULL;
2908         }
2909         c->ev->base = base;
2910         c->fd = -1;
2911         c->buffer = sldns_buffer_new(bufsize);
2912         if(!c->buffer) {
2913                 free(c->ev);
2914                 free(c);
2915                 return NULL;
2916         }
2917         c->timeout = NULL;
2918         c->tcp_is_reading = 0;
2919         c->tcp_byte_count = 0;
2920         c->tcp_parent = NULL;
2921         c->max_tcp_count = 0;
2922         c->cur_tcp_count = 0;
2923         c->tcp_handlers = NULL;
2924         c->tcp_free = NULL;
2925         c->type = comm_http;
2926         c->tcp_do_close = 0;
2927         c->do_not_close = 0;
2928         c->tcp_do_toggle_rw = 1;
2929         c->tcp_check_nb_connect = 1;
2930         c->http_in_headers = 1;
2931         c->http_in_chunk_headers = 0;
2932         c->http_is_chunked = 0;
2933         c->http_temp = temp;
2934 #ifdef USE_MSG_FASTOPEN
2935         c->tcp_do_fastopen = 1;
2936 #endif
2937 #ifdef USE_DNSCRYPT
2938         c->dnscrypt = 0;
2939         c->dnscrypt_buffer = c->buffer;
2940 #endif
2941         c->repinfo.c = c;
2942         c->callback = callback;
2943         c->cb_arg = callback_arg;
2944         evbits = UB_EV_PERSIST | UB_EV_WRITE;
2945         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2946                 comm_point_http_handle_callback, c);
2947         if(c->ev->ev == NULL)
2948         {
2949                 log_err("could not baseset tcpout event");
2950 #ifdef HAVE_SSL
2951                 SSL_free(c->ssl);
2952 #endif
2953                 sldns_buffer_free(c->buffer);
2954                 free(c->ev);
2955                 free(c);
2956                 return NULL;
2957         }
2958
2959         return c;
2960 }
2961
2962 struct comm_point*
2963 comm_point_create_local(struct comm_base *base, int fd, size_t bufsize,
2964         comm_point_callback_type* callback, void* callback_arg)
2965 {
2966         struct comm_point* c = (struct comm_point*)calloc(1,
2967                 sizeof(struct comm_point));
2968         short evbits;
2969         if(!c)
2970                 return NULL;
2971         c->ev = (struct internal_event*)calloc(1,
2972                 sizeof(struct internal_event));
2973         if(!c->ev) {
2974                 free(c);
2975                 return NULL;
2976         }
2977         c->ev->base = base;
2978         c->fd = fd;
2979         c->buffer = sldns_buffer_new(bufsize);
2980         if(!c->buffer) {
2981                 free(c->ev);
2982                 free(c);
2983                 return NULL;
2984         }
2985         c->timeout = NULL;
2986         c->tcp_is_reading = 1;
2987         c->tcp_byte_count = 0;
2988         c->tcp_parent = NULL;
2989         c->max_tcp_count = 0;
2990         c->cur_tcp_count = 0;
2991         c->tcp_handlers = NULL;
2992         c->tcp_free = NULL;
2993         c->type = comm_local;
2994         c->tcp_do_close = 0;
2995         c->do_not_close = 1;
2996         c->tcp_do_toggle_rw = 0;
2997         c->tcp_check_nb_connect = 0;
2998 #ifdef USE_MSG_FASTOPEN
2999         c->tcp_do_fastopen = 0;
3000 #endif
3001 #ifdef USE_DNSCRYPT
3002         c->dnscrypt = 0;
3003         c->dnscrypt_buffer = c->buffer;
3004 #endif
3005         c->callback = callback;
3006         c->cb_arg = callback_arg;
3007         /* ub_event stuff */
3008         evbits = UB_EV_PERSIST | UB_EV_READ;
3009         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3010                 comm_point_local_handle_callback, c);
3011         if(c->ev->ev == NULL) {
3012                 log_err("could not baseset localhdl event");
3013                 free(c->ev);
3014                 free(c);
3015                 return NULL;
3016         }
3017         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3018                 log_err("could not add localhdl event");
3019                 ub_event_free(c->ev->ev);
3020                 free(c->ev);
3021                 free(c);
3022                 return NULL;
3023         }
3024         return c;
3025 }
3026
3027 struct comm_point*
3028 comm_point_create_raw(struct comm_base* base, int fd, int writing,
3029         comm_point_callback_type* callback, void* callback_arg)
3030 {
3031         struct comm_point* c = (struct comm_point*)calloc(1,
3032                 sizeof(struct comm_point));
3033         short evbits;
3034         if(!c)
3035                 return NULL;
3036         c->ev = (struct internal_event*)calloc(1,
3037                 sizeof(struct internal_event));
3038         if(!c->ev) {
3039                 free(c);
3040                 return NULL;
3041         }
3042         c->ev->base = base;
3043         c->fd = fd;
3044         c->buffer = NULL;
3045         c->timeout = NULL;
3046         c->tcp_is_reading = 0;
3047         c->tcp_byte_count = 0;
3048         c->tcp_parent = NULL;
3049         c->max_tcp_count = 0;
3050         c->cur_tcp_count = 0;
3051         c->tcp_handlers = NULL;
3052         c->tcp_free = NULL;
3053         c->type = comm_raw;
3054         c->tcp_do_close = 0;
3055         c->do_not_close = 1;
3056         c->tcp_do_toggle_rw = 0;
3057         c->tcp_check_nb_connect = 0;
3058 #ifdef USE_MSG_FASTOPEN
3059         c->tcp_do_fastopen = 0;
3060 #endif
3061 #ifdef USE_DNSCRYPT
3062         c->dnscrypt = 0;
3063         c->dnscrypt_buffer = c->buffer;
3064 #endif
3065         c->callback = callback;
3066         c->cb_arg = callback_arg;
3067         /* ub_event stuff */
3068         if(writing)
3069                 evbits = UB_EV_PERSIST | UB_EV_WRITE;
3070         else    evbits = UB_EV_PERSIST | UB_EV_READ;
3071         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3072                 comm_point_raw_handle_callback, c);
3073         if(c->ev->ev == NULL) {
3074                 log_err("could not baseset rawhdl event");
3075                 free(c->ev);
3076                 free(c);
3077                 return NULL;
3078         }
3079         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3080                 log_err("could not add rawhdl event");
3081                 ub_event_free(c->ev->ev);
3082                 free(c->ev);
3083                 free(c);
3084                 return NULL;
3085         }
3086         return c;
3087 }
3088
3089 void
3090 comm_point_close(struct comm_point* c)
3091 {
3092         if(!c)
3093                 return;
3094         if(c->fd != -1) {
3095                 if(ub_event_del(c->ev->ev) != 0) {
3096                         log_err("could not event_del on close");
3097                 }
3098         }
3099         tcl_close_connection(c->tcl_addr);
3100         if(c->tcp_req_info)
3101                 tcp_req_info_clear(c->tcp_req_info);
3102         /* close fd after removing from event lists, or epoll.. is messed up */
3103         if(c->fd != -1 && !c->do_not_close) {
3104                 if(c->type == comm_tcp || c->type == comm_http) {
3105                         /* delete sticky events for the fd, it gets closed */
3106                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
3107                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
3108                 }
3109                 verbose(VERB_ALGO, "close fd %d", c->fd);
3110 #ifndef USE_WINSOCK
3111                 close(c->fd);
3112 #else
3113                 closesocket(c->fd);
3114 #endif
3115         }
3116         c->fd = -1;
3117 }
3118
3119 void
3120 comm_point_delete(struct comm_point* c)
3121 {
3122         if(!c)
3123                 return;
3124         if((c->type == comm_tcp || c->type == comm_http) && c->ssl) {
3125 #ifdef HAVE_SSL
3126                 SSL_shutdown(c->ssl);
3127                 SSL_free(c->ssl);
3128 #endif
3129         }
3130         comm_point_close(c);
3131         if(c->tcp_handlers) {
3132                 int i;
3133                 for(i=0; i<c->max_tcp_count; i++)
3134                         comm_point_delete(c->tcp_handlers[i]);
3135                 free(c->tcp_handlers);
3136         }
3137         free(c->timeout);
3138         if(c->type == comm_tcp || c->type == comm_local || c->type == comm_http) {
3139                 sldns_buffer_free(c->buffer);
3140 #ifdef USE_DNSCRYPT
3141                 if(c->dnscrypt && c->dnscrypt_buffer != c->buffer) {
3142                         sldns_buffer_free(c->dnscrypt_buffer);
3143                 }
3144 #endif
3145                 if(c->tcp_req_info) {
3146                         tcp_req_info_delete(c->tcp_req_info);
3147                 }
3148         }
3149         ub_event_free(c->ev->ev);
3150         free(c->ev);
3151         free(c);
3152 }
3153
3154 void
3155 comm_point_send_reply(struct comm_reply *repinfo)
3156 {
3157         struct sldns_buffer* buffer;
3158         log_assert(repinfo && repinfo->c);
3159 #ifdef USE_DNSCRYPT
3160         buffer = repinfo->c->dnscrypt_buffer;
3161         if(!dnsc_handle_uncurved_request(repinfo)) {
3162                 return;
3163         }
3164 #else
3165         buffer = repinfo->c->buffer;
3166 #endif
3167         if(repinfo->c->type == comm_udp) {
3168                 if(repinfo->srctype)
3169                         comm_point_send_udp_msg_if(repinfo->c,
3170                         buffer, (struct sockaddr*)&repinfo->addr,
3171                         repinfo->addrlen, repinfo);
3172                 else
3173                         comm_point_send_udp_msg(repinfo->c, buffer,
3174                         (struct sockaddr*)&repinfo->addr, repinfo->addrlen);
3175 #ifdef USE_DNSTAP
3176                 if(repinfo->c->dtenv != NULL &&
3177                    repinfo->c->dtenv->log_client_response_messages)
3178                         dt_msg_send_client_response(repinfo->c->dtenv,
3179                         &repinfo->addr, repinfo->c->type, repinfo->c->buffer);
3180 #endif
3181         } else {
3182 #ifdef USE_DNSTAP
3183                 if(repinfo->c->tcp_parent->dtenv != NULL &&
3184                    repinfo->c->tcp_parent->dtenv->log_client_response_messages)
3185                         dt_msg_send_client_response(repinfo->c->tcp_parent->dtenv,
3186                         &repinfo->addr, repinfo->c->type, repinfo->c->buffer);
3187 #endif
3188                 if(repinfo->c->tcp_req_info) {
3189                         tcp_req_info_send_reply(repinfo->c->tcp_req_info);
3190                 } else {
3191                         comm_point_start_listening(repinfo->c, -1,
3192                                 repinfo->c->tcp_timeout_msec);
3193                 }
3194         }
3195 }
3196
3197 void
3198 comm_point_drop_reply(struct comm_reply* repinfo)
3199 {
3200         if(!repinfo)
3201                 return;
3202         log_assert(repinfo->c);
3203         log_assert(repinfo->c->type != comm_tcp_accept);
3204         if(repinfo->c->type == comm_udp)
3205                 return;
3206         if(repinfo->c->tcp_req_info)
3207                 repinfo->c->tcp_req_info->is_drop = 1;
3208         reclaim_tcp_handler(repinfo->c);
3209 }
3210
3211 void
3212 comm_point_stop_listening(struct comm_point* c)
3213 {
3214         verbose(VERB_ALGO, "comm point stop listening %d", c->fd);
3215         if(ub_event_del(c->ev->ev) != 0) {
3216                 log_err("event_del error to stoplisten");
3217         }
3218 }
3219
3220 void
3221 comm_point_start_listening(struct comm_point* c, int newfd, int msec)
3222 {
3223         verbose(VERB_ALGO, "comm point start listening %d (%d msec)",
3224                 c->fd==-1?newfd:c->fd, msec);
3225         if(c->type == comm_tcp_accept && !c->tcp_free) {
3226                 /* no use to start listening no free slots. */
3227                 return;
3228         }
3229         if(msec != -1 && msec != 0) {
3230                 if(!c->timeout) {
3231                         c->timeout = (struct timeval*)malloc(sizeof(
3232                                 struct timeval));
3233                         if(!c->timeout) {
3234                                 log_err("cpsl: malloc failed. No net read.");
3235                                 return;
3236                         }
3237                 }
3238                 ub_event_add_bits(c->ev->ev, UB_EV_TIMEOUT);
3239 #ifndef S_SPLINT_S /* splint fails on struct timeval. */
3240                 c->timeout->tv_sec = msec/1000;
3241                 c->timeout->tv_usec = (msec%1000)*1000;
3242 #endif /* S_SPLINT_S */
3243         }
3244         if(c->type == comm_tcp || c->type == comm_http) {
3245                 ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
3246                 if(c->tcp_is_reading)
3247                         ub_event_add_bits(c->ev->ev, UB_EV_READ);
3248                 else    ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
3249         }
3250         if(newfd != -1) {
3251                 if(c->fd != -1) {
3252 #ifndef USE_WINSOCK
3253                         close(c->fd);
3254 #else
3255                         closesocket(c->fd);
3256 #endif
3257                 }
3258                 c->fd = newfd;
3259                 ub_event_set_fd(c->ev->ev, c->fd);
3260         }
3261         if(ub_event_add(c->ev->ev, msec==0?NULL:c->timeout) != 0) {
3262                 log_err("event_add failed. in cpsl.");
3263         }
3264 }
3265
3266 void comm_point_listen_for_rw(struct comm_point* c, int rd, int wr)
3267 {
3268         verbose(VERB_ALGO, "comm point listen_for_rw %d %d", c->fd, wr);
3269         if(ub_event_del(c->ev->ev) != 0) {
3270                 log_err("event_del error to cplf");
3271         }
3272         ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
3273         if(rd) ub_event_add_bits(c->ev->ev, UB_EV_READ);
3274         if(wr) ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
3275         if(ub_event_add(c->ev->ev, c->timeout) != 0) {
3276                 log_err("event_add failed. in cplf.");
3277         }
3278 }
3279
3280 size_t comm_point_get_mem(struct comm_point* c)
3281 {
3282         size_t s;
3283         if(!c)
3284                 return 0;
3285         s = sizeof(*c) + sizeof(*c->ev);
3286         if(c->timeout)
3287                 s += sizeof(*c->timeout);
3288         if(c->type == comm_tcp || c->type == comm_local) {
3289                 s += sizeof(*c->buffer) + sldns_buffer_capacity(c->buffer);
3290 #ifdef USE_DNSCRYPT
3291                 s += sizeof(*c->dnscrypt_buffer);
3292                 if(c->buffer != c->dnscrypt_buffer) {
3293                         s += sldns_buffer_capacity(c->dnscrypt_buffer);
3294                 }
3295 #endif
3296         }
3297         if(c->type == comm_tcp_accept) {
3298                 int i;
3299                 for(i=0; i<c->max_tcp_count; i++)
3300                         s += comm_point_get_mem(c->tcp_handlers[i]);
3301         }
3302         return s;
3303 }
3304
3305 struct comm_timer*
3306 comm_timer_create(struct comm_base* base, void (*cb)(void*), void* cb_arg)
3307 {
3308         struct internal_timer *tm = (struct internal_timer*)calloc(1,
3309                 sizeof(struct internal_timer));
3310         if(!tm) {
3311                 log_err("malloc failed");
3312                 return NULL;
3313         }
3314         tm->super.ev_timer = tm;
3315         tm->base = base;
3316         tm->super.callback = cb;
3317         tm->super.cb_arg = cb_arg;
3318         tm->ev = ub_event_new(base->eb->base, -1, UB_EV_TIMEOUT,
3319                 comm_timer_callback, &tm->super);
3320         if(tm->ev == NULL) {
3321                 log_err("timer_create: event_base_set failed.");
3322                 free(tm);
3323                 return NULL;
3324         }
3325         return &tm->super;
3326 }
3327
3328 void
3329 comm_timer_disable(struct comm_timer* timer)
3330 {
3331         if(!timer)
3332                 return;
3333         ub_timer_del(timer->ev_timer->ev);
3334         timer->ev_timer->enabled = 0;
3335 }
3336
3337 void
3338 comm_timer_set(struct comm_timer* timer, struct timeval* tv)
3339 {
3340         log_assert(tv);
3341         if(timer->ev_timer->enabled)
3342                 comm_timer_disable(timer);
3343         if(ub_timer_add(timer->ev_timer->ev, timer->ev_timer->base->eb->base,
3344                 comm_timer_callback, timer, tv) != 0)
3345                 log_err("comm_timer_set: evtimer_add failed.");
3346         timer->ev_timer->enabled = 1;
3347 }
3348
3349 void
3350 comm_timer_delete(struct comm_timer* timer)
3351 {
3352         if(!timer)
3353                 return;
3354         comm_timer_disable(timer);
3355         /* Free the sub struct timer->ev_timer derived from the super struct timer.
3356          * i.e. assert(timer == timer->ev_timer)
3357          */
3358         ub_event_free(timer->ev_timer->ev);
3359         free(timer->ev_timer);
3360 }
3361
3362 void
3363 comm_timer_callback(int ATTR_UNUSED(fd), short event, void* arg)
3364 {
3365         struct comm_timer* tm = (struct comm_timer*)arg;
3366         if(!(event&UB_EV_TIMEOUT))
3367                 return;
3368         ub_comm_base_now(tm->ev_timer->base);
3369         tm->ev_timer->enabled = 0;
3370         fptr_ok(fptr_whitelist_comm_timer(tm->callback));
3371         (*tm->callback)(tm->cb_arg);
3372 }
3373
3374 int
3375 comm_timer_is_set(struct comm_timer* timer)
3376 {
3377         return (int)timer->ev_timer->enabled;
3378 }
3379
3380 size_t
3381 comm_timer_get_mem(struct comm_timer* ATTR_UNUSED(timer))
3382 {
3383         return sizeof(struct internal_timer);
3384 }
3385
3386 struct comm_signal*
3387 comm_signal_create(struct comm_base* base,
3388         void (*callback)(int, void*), void* cb_arg)
3389 {
3390         struct comm_signal* com = (struct comm_signal*)malloc(
3391                 sizeof(struct comm_signal));
3392         if(!com) {
3393                 log_err("malloc failed");
3394                 return NULL;
3395         }
3396         com->base = base;
3397         com->callback = callback;
3398         com->cb_arg = cb_arg;
3399         com->ev_signal = NULL;
3400         return com;
3401 }
3402
3403 void
3404 comm_signal_callback(int sig, short event, void* arg)
3405 {
3406         struct comm_signal* comsig = (struct comm_signal*)arg;
3407         if(!(event & UB_EV_SIGNAL))
3408                 return;
3409         ub_comm_base_now(comsig->base);
3410         fptr_ok(fptr_whitelist_comm_signal(comsig->callback));
3411         (*comsig->callback)(sig, comsig->cb_arg);
3412 }
3413
3414 int
3415 comm_signal_bind(struct comm_signal* comsig, int sig)
3416 {
3417         struct internal_signal* entry = (struct internal_signal*)calloc(1,
3418                 sizeof(struct internal_signal));
3419         if(!entry) {
3420                 log_err("malloc failed");
3421                 return 0;
3422         }
3423         log_assert(comsig);
3424         /* add signal event */
3425         entry->ev = ub_signal_new(comsig->base->eb->base, sig,
3426                 comm_signal_callback, comsig);
3427         if(entry->ev == NULL) {
3428                 log_err("Could not create signal event");
3429                 free(entry);
3430                 return 0;
3431         }
3432         if(ub_signal_add(entry->ev, NULL) != 0) {
3433                 log_err("Could not add signal handler");
3434                 ub_event_free(entry->ev);
3435                 free(entry);
3436                 return 0;
3437         }
3438         /* link into list */
3439         entry->next = comsig->ev_signal;
3440         comsig->ev_signal = entry;
3441         return 1;
3442 }
3443
3444 void
3445 comm_signal_delete(struct comm_signal* comsig)
3446 {
3447         struct internal_signal* p, *np;
3448         if(!comsig)
3449                 return;
3450         p=comsig->ev_signal;
3451         while(p) {
3452                 np = p->next;
3453                 ub_signal_del(p->ev);
3454                 ub_event_free(p->ev);
3455                 free(p);
3456                 p = np;
3457         }
3458         free(comsig);
3459 }