contrib/unbound/util/netevent.c

   1 /*
   2  * util/netevent.c - event notification
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains event notification functions.
  40  */
  41 #include "config.h"
  42 #include "util/netevent.h"
  43 #include "util/ub_event.h"
  44 #include "util/log.h"
  45 #include "util/net_help.h"
  46 #include "util/tcp_conn_limit.h"
  47 #include "util/fptr_wlist.h"
  48 #include "sldns/pkthdr.h"
  49 #include "sldns/sbuffer.h"
  50 #include "sldns/str2wire.h"
  51 #include "dnstap/dnstap.h"
  52 #include "dnscrypt/dnscrypt.h"
  53 #include "services/listen_dnsport.h"
  54 #ifdef HAVE_OPENSSL_SSL_H
  55 #include <openssl/ssl.h>
  56 #endif
  57 #ifdef HAVE_OPENSSL_ERR_H
  58 #include <openssl/err.h>
  59 #endif
  60
  61 /* -------- Start of local definitions -------- */
  62 /** if CMSG_ALIGN is not defined on this platform, a workaround */
  63 #ifndef CMSG_ALIGN
  64 #  ifdef __CMSG_ALIGN
  65 #    define CMSG_ALIGN(n) __CMSG_ALIGN(n)
  66 #  elif defined(CMSG_DATA_ALIGN)
  67 #    define CMSG_ALIGN _CMSG_DATA_ALIGN
  68 #  else
  69 #    define CMSG_ALIGN(len) (((len)+sizeof(long)-1) & ~(sizeof(long)-1))
  70 #  endif
  71 #endif
  72
  73 /** if CMSG_LEN is not defined on this platform, a workaround */
  74 #ifndef CMSG_LEN
  75 #  define CMSG_LEN(len) (CMSG_ALIGN(sizeof(struct cmsghdr))+(len))
  76 #endif
  77
  78 /** if CMSG_SPACE is not defined on this platform, a workaround */
  79 #ifndef CMSG_SPACE
  80 #  ifdef _CMSG_HDR_ALIGN
  81 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+_CMSG_HDR_ALIGN(sizeof(struct cmsghdr)))
  82 #  else
  83 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+CMSG_ALIGN(sizeof(struct cmsghdr)))
  84 #  endif
  85 #endif
  86
  87 /** The TCP writing query timeout in milliseconds */
  88 #define TCP_QUERY_TIMEOUT 120000
  89 /** The minimum actual TCP timeout to use, regardless of what we advertise,
  90  * in msec */
  91 #define TCP_QUERY_TIMEOUT_MINIMUM 200
  92
  93 #ifndef NONBLOCKING_IS_BROKEN
  94 /** number of UDP reads to perform per read indication from select */
  95 #define NUM_UDP_PER_SELECT 100
  96 #else
  97 #define NUM_UDP_PER_SELECT 1
  98 #endif
  99
 100 /**
 101  * The internal event structure for keeping ub_event info for the event.
 102  * Possibly other structures (list, tree) this is part of.
 103  */
 104 struct internal_event {
 105         /** the comm base */
 106         struct comm_base* base;
 107         /** ub_event event type */
 108         struct ub_event* ev;
 109 };
 110
 111 /**
 112  * Internal base structure, so that every thread has its own events.
 113  */
 114 struct internal_base {
 115         /** ub_event event_base type. */
 116         struct ub_event_base* base;
 117         /** seconds time pointer points here */
 118         time_t secs;
 119         /** timeval with current time */
 120         struct timeval now;
 121         /** the event used for slow_accept timeouts */
 122         struct ub_event* slow_accept;
 123         /** true if slow_accept is enabled */
 124         int slow_accept_enabled;
 125 };
 126
 127 /**
 128  * Internal timer structure, to store timer event in.
 129  */
 130 struct internal_timer {
 131         /** the super struct from which derived */
 132         struct comm_timer super;
 133         /** the comm base */
 134         struct comm_base* base;
 135         /** ub_event event type */
 136         struct ub_event* ev;
 137         /** is timer enabled */
 138         uint8_t enabled;
 139 };
 140
 141 /**
 142  * Internal signal structure, to store signal event in.
 143  */
 144 struct internal_signal {
 145         /** ub_event event type */
 146         struct ub_event* ev;
 147         /** next in signal list */
 148         struct internal_signal* next;
 149 };
 150
 151 /** create a tcp handler with a parent */
 152 static struct comm_point* comm_point_create_tcp_handler(
 153         struct comm_base *base, struct comm_point* parent, size_t bufsize,
 154         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
 155         void* callback_arg);
 156
 157 /* -------- End of local definitions -------- */
 158
 159 struct comm_base*
 160 comm_base_create(int sigs)
 161 {
 162         struct comm_base* b = (struct comm_base*)calloc(1,
 163                 sizeof(struct comm_base));
 164         const char *evnm="event", *evsys="", *evmethod="";
 165
 166         if(!b)
 167                 return NULL;
 168         b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
 169         if(!b->eb) {
 170                 free(b);
 171                 return NULL;
 172         }
 173         b->eb->base = ub_default_event_base(sigs, &b->eb->secs, &b->eb->now);
 174         if(!b->eb->base) {
 175                 free(b->eb);
 176                 free(b);
 177                 return NULL;
 178         }
 179         ub_comm_base_now(b);
 180         ub_get_event_sys(b->eb->base, &evnm, &evsys, &evmethod);
 181         verbose(VERB_ALGO, "%s %s uses %s method.", evnm, evsys, evmethod);
 182         return b;
 183 }
 184
 185 struct comm_base*
 186 comm_base_create_event(struct ub_event_base* base)
 187 {
 188         struct comm_base* b = (struct comm_base*)calloc(1,
 189                 sizeof(struct comm_base));
 190         if(!b)
 191                 return NULL;
 192         b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
 193         if(!b->eb) {
 194                 free(b);
 195                 return NULL;
 196         }
 197         b->eb->base = base;
 198         ub_comm_base_now(b);
 199         return b;
 200 }
 201
 202 void
 203 comm_base_delete(struct comm_base* b)
 204 {
 205         if(!b)
 206                 return;
 207         if(b->eb->slow_accept_enabled) {
 208                 if(ub_event_del(b->eb->slow_accept) != 0) {
 209                         log_err("could not event_del slow_accept");
 210                 }
 211                 ub_event_free(b->eb->slow_accept);
 212         }
 213         ub_event_base_free(b->eb->base);
 214         b->eb->base = NULL;
 215         free(b->eb);
 216         free(b);
 217 }
 218
 219 void
 220 comm_base_delete_no_base(struct comm_base* b)
 221 {
 222         if(!b)
 223                 return;
 224         if(b->eb->slow_accept_enabled) {
 225                 if(ub_event_del(b->eb->slow_accept) != 0) {
 226                         log_err("could not event_del slow_accept");
 227                 }
 228                 ub_event_free(b->eb->slow_accept);
 229         }
 230         b->eb->base = NULL;
 231         free(b->eb);
 232         free(b);
 233 }
 234
 235 void
 236 comm_base_timept(struct comm_base* b, time_t** tt, struct timeval** tv)
 237 {
 238         *tt = &b->eb->secs;
 239         *tv = &b->eb->now;
 240 }
 241
 242 void
 243 comm_base_dispatch(struct comm_base* b)
 244 {
 245         int retval;
 246         retval = ub_event_base_dispatch(b->eb->base);
 247         if(retval < 0) {
 248                 fatal_exit("event_dispatch returned error %d, "
 249                         "errno is %s", retval, strerror(errno));
 250         }
 251 }
 252
 253 void comm_base_exit(struct comm_base* b)
 254 {
 255         if(ub_event_base_loopexit(b->eb->base) != 0) {
 256                 log_err("Could not loopexit");
 257         }
 258 }
 259
 260 void comm_base_set_slow_accept_handlers(struct comm_base* b,
 261         void (*stop_acc)(void*), void (*start_acc)(void*), void* arg)
 262 {
 263         b->stop_accept = stop_acc;
 264         b->start_accept = start_acc;
 265         b->cb_arg = arg;
 266 }
 267
 268 struct ub_event_base* comm_base_internal(struct comm_base* b)
 269 {
 270         return b->eb->base;
 271 }
 272
 273 /** see if errno for udp has to be logged or not uses globals */
 274 static int
 275 udp_send_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
 276 {
 277         /* do not log transient errors (unless high verbosity) */
 278 #if defined(ENETUNREACH) || defined(EHOSTDOWN) || defined(EHOSTUNREACH) || defined(ENETDOWN)
 279         switch(errno) {
 280 #  ifdef ENETUNREACH
 281                 case ENETUNREACH:
 282 #  endif
 283 #  ifdef EHOSTDOWN
 284                 case EHOSTDOWN:
 285 #  endif
 286 #  ifdef EHOSTUNREACH
 287                 case EHOSTUNREACH:
 288 #  endif
 289 #  ifdef ENETDOWN
 290                 case ENETDOWN:
 291 #  endif
 292                         if(verbosity < VERB_ALGO)
 293                                 return 0;
 294                 default:
 295                         break;
 296         }
 297 #endif
 298         /* permission denied is gotten for every send if the
 299          * network is disconnected (on some OS), squelch it */
 300         if( ((errno == EPERM)
 301 #  ifdef EADDRNOTAVAIL
 302                 /* 'Cannot assign requested address' also when disconnected */
 303                 || (errno == EADDRNOTAVAIL)
 304 #  endif
 305                 ) && verbosity < VERB_DETAIL)
 306                 return 0;
 307 #  ifdef EADDRINUSE
 308         /* If SO_REUSEADDR is set, we could try to connect to the same server
 309          * from the same source port twice. */
 310         if(errno == EADDRINUSE && verbosity < VERB_DETAIL)
 311                 return 0;
 312 #  endif
 313         /* squelch errors where people deploy AAAA ::ffff:bla for
 314          * authority servers, which we try for intranets. */
 315         if(errno == EINVAL && addr_is_ip4mapped(
 316                 (struct sockaddr_storage*)addr, addrlen) &&
 317                 verbosity < VERB_DETAIL)
 318                 return 0;
 319         /* SO_BROADCAST sockopt can give access to 255.255.255.255,
 320          * but a dns cache does not need it. */
 321         if(errno == EACCES && addr_is_broadcast(
 322                 (struct sockaddr_storage*)addr, addrlen) &&
 323                 verbosity < VERB_DETAIL)
 324                 return 0;
 325         return 1;
 326 }
 327
 328 int tcp_connect_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
 329 {
 330         return udp_send_errno_needs_log(addr, addrlen);
 331 }
 332
 333 /* send a UDP reply */
 334 int
 335 comm_point_send_udp_msg(struct comm_point *c, sldns_buffer* packet,
 336         struct sockaddr* addr, socklen_t addrlen)
 337 {
 338         ssize_t sent;
 339         log_assert(c->fd != -1);
 340 #ifdef UNBOUND_DEBUG
 341         if(sldns_buffer_remaining(packet) == 0)
 342                 log_err("error: send empty UDP packet");
 343 #endif
 344         log_assert(addr && addrlen > 0);
 345         sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
 346                 sldns_buffer_remaining(packet), 0,
 347                 addr, addrlen);
 348         if(sent == -1) {
 349                 /* try again and block, waiting for IO to complete,
 350                  * we want to send the answer, and we will wait for
 351                  * the ethernet interface buffer to have space. */
 352 #ifndef USE_WINSOCK
 353                 if(errno == EAGAIN ||
 354 #  ifdef EWOULDBLOCK
 355                         errno == EWOULDBLOCK ||
 356 #  endif
 357                         errno == ENOBUFS) {
 358 #else
 359                 if(WSAGetLastError() == WSAEINPROGRESS ||
 360                         WSAGetLastError() == WSAENOBUFS ||
 361                         WSAGetLastError() == WSAEWOULDBLOCK) {
 362 #endif
 363                         int e;
 364                         fd_set_block(c->fd);
 365                         sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
 366                                 sldns_buffer_remaining(packet), 0,
 367                                 addr, addrlen);
 368                         e = errno;
 369                         fd_set_nonblock(c->fd);
 370                         errno = e;
 371                 }
 372         }
 373         if(sent == -1) {
 374                 if(!udp_send_errno_needs_log(addr, addrlen))
 375                         return 0;
 376                 verbose(VERB_OPS, "sendto failed: %s", sock_strerror(errno));
 377                 log_addr(VERB_OPS, "remote address is",
 378                         (struct sockaddr_storage*)addr, addrlen);
 379                 return 0;
 380         } else if((size_t)sent != sldns_buffer_remaining(packet)) {
 381                 log_err("sent %d in place of %d bytes",
 382                         (int)sent, (int)sldns_buffer_remaining(packet));
 383                 return 0;
 384         }
 385         return 1;
 386 }
 387
 388 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && (defined(HAVE_RECVMSG) || defined(HAVE_SENDMSG))
 389 /** print debug ancillary info */
 390 static void p_ancil(const char* str, struct comm_reply* r)
 391 {
 392         if(r->srctype != 4 && r->srctype != 6) {
 393                 log_info("%s: unknown srctype %d", str, r->srctype);
 394                 return;
 395         }
 396         if(r->srctype == 6) {
 397                 char buf[1024];
 398                 if(inet_ntop(AF_INET6, &r->pktinfo.v6info.ipi6_addr,
 399                         buf, (socklen_t)sizeof(buf)) == 0) {
 400                         (void)strlcpy(buf, "(inet_ntop error)", sizeof(buf));
 401                 }
 402                 buf[sizeof(buf)-1]=0;
 403                 log_info("%s: %s %d", str, buf, r->pktinfo.v6info.ipi6_ifindex);
 404         } else if(r->srctype == 4) {
 405 #ifdef IP_PKTINFO
 406                 char buf1[1024], buf2[1024];
 407                 if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_addr,
 408                         buf1, (socklen_t)sizeof(buf1)) == 0) {
 409                         (void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
 410                 }
 411                 buf1[sizeof(buf1)-1]=0;
 412 #ifdef HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST
 413                 if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_spec_dst,
 414                         buf2, (socklen_t)sizeof(buf2)) == 0) {
 415                         (void)strlcpy(buf2, "(inet_ntop error)", sizeof(buf2));
 416                 }
 417                 buf2[sizeof(buf2)-1]=0;
 418 #else
 419                 buf2[0]=0;
 420 #endif
 421                 log_info("%s: %d %s %s", str, r->pktinfo.v4info.ipi_ifindex,
 422                         buf1, buf2);
 423 #elif defined(IP_RECVDSTADDR)
 424                 char buf1[1024];
 425                 if(inet_ntop(AF_INET, &r->pktinfo.v4addr,
 426                         buf1, (socklen_t)sizeof(buf1)) == 0) {
 427                         (void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
 428                 }
 429                 buf1[sizeof(buf1)-1]=0;
 430                 log_info("%s: %s", str, buf1);
 431 #endif /* IP_PKTINFO or PI_RECVDSTDADDR */
 432         }
 433 }
 434 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG||HAVE_SENDMSG */
 435
 436 /** send a UDP reply over specified interface*/
 437 static int
 438 comm_point_send_udp_msg_if(struct comm_point *c, sldns_buffer* packet,
 439         struct sockaddr* addr, socklen_t addrlen, struct comm_reply* r)
 440 {
 441 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_SENDMSG)
 442         ssize_t sent;
 443         struct msghdr msg;
 444         struct iovec iov[1];
 445         union {
 446                 struct cmsghdr hdr;
 447                 char buf[256];
 448         } control;
 449 #ifndef S_SPLINT_S
 450         struct cmsghdr *cmsg;
 451 #endif /* S_SPLINT_S */
 452
 453         log_assert(c->fd != -1);
 454 #ifdef UNBOUND_DEBUG
 455         if(sldns_buffer_remaining(packet) == 0)
 456                 log_err("error: send empty UDP packet");
 457 #endif
 458         log_assert(addr && addrlen > 0);
 459
 460         msg.msg_name = addr;
 461         msg.msg_namelen = addrlen;
 462         iov[0].iov_base = sldns_buffer_begin(packet);
 463         iov[0].iov_len = sldns_buffer_remaining(packet);
 464         msg.msg_iov = iov;
 465         msg.msg_iovlen = 1;
 466         msg.msg_control = control.buf;
 467 #ifndef S_SPLINT_S
 468         msg.msg_controllen = sizeof(control.buf);
 469 #endif /* S_SPLINT_S */
 470         msg.msg_flags = 0;
 471
 472 #ifndef S_SPLINT_S
 473         cmsg = CMSG_FIRSTHDR(&msg);
 474         if(r->srctype == 4) {
 475 #ifdef IP_PKTINFO
 476                 void* cmsg_data;
 477                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo));
 478                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 479                 cmsg->cmsg_level = IPPROTO_IP;
 480                 cmsg->cmsg_type = IP_PKTINFO;
 481                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v4info,
 482                         sizeof(struct in_pktinfo));
 483                 /* unset the ifindex to not bypass the routing tables */
 484                 cmsg_data = CMSG_DATA(cmsg);
 485                 ((struct in_pktinfo *) cmsg_data)->ipi_ifindex = 0;
 486                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
 487 #elif defined(IP_SENDSRCADDR)
 488                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in_addr));
 489                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 490                 cmsg->cmsg_level = IPPROTO_IP;
 491                 cmsg->cmsg_type = IP_SENDSRCADDR;
 492                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v4addr,
 493                         sizeof(struct in_addr));
 494                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_addr));
 495 #else
 496                 verbose(VERB_ALGO, "no IP_PKTINFO or IP_SENDSRCADDR");
 497                 msg.msg_control = NULL;
 498 #endif /* IP_PKTINFO or IP_SENDSRCADDR */
 499         } else if(r->srctype == 6) {
 500                 void* cmsg_data;
 501                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
 502                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 503                 cmsg->cmsg_level = IPPROTO_IPV6;
 504                 cmsg->cmsg_type = IPV6_PKTINFO;
 505                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v6info,
 506                         sizeof(struct in6_pktinfo));
 507                 /* unset the ifindex to not bypass the routing tables */
 508                 cmsg_data = CMSG_DATA(cmsg);
 509                 ((struct in6_pktinfo *) cmsg_data)->ipi6_ifindex = 0;
 510                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
 511         } else {
 512                 /* try to pass all 0 to use default route */
 513                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
 514                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 515                 cmsg->cmsg_level = IPPROTO_IPV6;
 516                 cmsg->cmsg_type = IPV6_PKTINFO;
 517                 memset(CMSG_DATA(cmsg), 0, sizeof(struct in6_pktinfo));
 518                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
 519         }
 520 #endif /* S_SPLINT_S */
 521         if(verbosity >= VERB_ALGO)
 522                 p_ancil("send_udp over interface", r);
 523         sent = sendmsg(c->fd, &msg, 0);
 524         if(sent == -1) {
 525                 /* try again and block, waiting for IO to complete,
 526                  * we want to send the answer, and we will wait for
 527                  * the ethernet interface buffer to have space. */
 528 #ifndef USE_WINSOCK
 529                 if(errno == EAGAIN ||
 530 #  ifdef EWOULDBLOCK
 531                         errno == EWOULDBLOCK ||
 532 #  endif
 533                         errno == ENOBUFS) {
 534 #else
 535                 if(WSAGetLastError() == WSAEINPROGRESS ||
 536                         WSAGetLastError() == WSAENOBUFS ||
 537                         WSAGetLastError() == WSAEWOULDBLOCK) {
 538 #endif
 539                         int e;
 540                         fd_set_block(c->fd);
 541                         sent = sendmsg(c->fd, &msg, 0);
 542                         e = errno;
 543                         fd_set_nonblock(c->fd);
 544                         errno = e;
 545                 }
 546         }
 547         if(sent == -1) {
 548                 if(!udp_send_errno_needs_log(addr, addrlen))
 549                         return 0;
 550                 verbose(VERB_OPS, "sendmsg failed: %s", strerror(errno));
 551                 log_addr(VERB_OPS, "remote address is",
 552                         (struct sockaddr_storage*)addr, addrlen);
 553 #ifdef __NetBSD__
 554                 /* netbsd 7 has IP_PKTINFO for recv but not send */
 555                 if(errno == EINVAL && r->srctype == 4)
 556                         log_err("sendmsg: No support for sendmsg(IP_PKTINFO). "
 557                                 "Please disable interface-automatic");
 558 #endif
 559                 return 0;
 560         } else if((size_t)sent != sldns_buffer_remaining(packet)) {
 561                 log_err("sent %d in place of %d bytes",
 562                         (int)sent, (int)sldns_buffer_remaining(packet));
 563                 return 0;
 564         }
 565         return 1;
 566 #else
 567         (void)c;
 568         (void)packet;
 569         (void)addr;
 570         (void)addrlen;
 571         (void)r;
 572         log_err("sendmsg: IPV6_PKTINFO not supported");
 573         return 0;
 574 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_SENDMSG */
 575 }
 576
 577 void
 578 comm_point_udp_ancil_callback(int fd, short event, void* arg)
 579 {
 580 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_RECVMSG)
 581         struct comm_reply rep;
 582         struct msghdr msg;
 583         struct iovec iov[1];
 584         ssize_t rcv;
 585         union {
 586                 struct cmsghdr hdr;
 587                 char buf[256];
 588         } ancil;
 589         int i;
 590 #ifndef S_SPLINT_S
 591         struct cmsghdr* cmsg;
 592 #endif /* S_SPLINT_S */
 593
 594         rep.c = (struct comm_point*)arg;
 595         log_assert(rep.c->type == comm_udp);
 596
 597         if(!(event&UB_EV_READ))
 598                 return;
 599         log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
 600         ub_comm_base_now(rep.c->ev->base);
 601         for(i=0; i<NUM_UDP_PER_SELECT; i++) {
 602                 sldns_buffer_clear(rep.c->buffer);
 603                 rep.addrlen = (socklen_t)sizeof(rep.addr);
 604                 log_assert(fd != -1);
 605                 log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
 606                 msg.msg_name = &rep.addr;
 607                 msg.msg_namelen = (socklen_t)sizeof(rep.addr);
 608                 iov[0].iov_base = sldns_buffer_begin(rep.c->buffer);
 609                 iov[0].iov_len = sldns_buffer_remaining(rep.c->buffer);
 610                 msg.msg_iov = iov;
 611                 msg.msg_iovlen = 1;
 612                 msg.msg_control = ancil.buf;
 613 #ifndef S_SPLINT_S
 614                 msg.msg_controllen = sizeof(ancil.buf);
 615 #endif /* S_SPLINT_S */
 616                 msg.msg_flags = 0;
 617                 rcv = recvmsg(fd, &msg, 0);
 618                 if(rcv == -1) {
 619                         if(errno != EAGAIN && errno != EINTR) {
 620                                 log_err("recvmsg failed: %s", strerror(errno));
 621                         }
 622                         return;
 623                 }
 624                 rep.addrlen = msg.msg_namelen;
 625                 sldns_buffer_skip(rep.c->buffer, rcv);
 626                 sldns_buffer_flip(rep.c->buffer);
 627                 rep.srctype = 0;
 628 #ifndef S_SPLINT_S
 629                 for(cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
 630                         cmsg = CMSG_NXTHDR(&msg, cmsg)) {
 631                         if( cmsg->cmsg_level == IPPROTO_IPV6 &&
 632                                 cmsg->cmsg_type == IPV6_PKTINFO) {
 633                                 rep.srctype = 6;
 634                                 memmove(&rep.pktinfo.v6info, CMSG_DATA(cmsg),
 635                                         sizeof(struct in6_pktinfo));
 636                                 break;
 637 #ifdef IP_PKTINFO
 638                         } else if( cmsg->cmsg_level == IPPROTO_IP &&
 639                                 cmsg->cmsg_type == IP_PKTINFO) {
 640                                 rep.srctype = 4;
 641                                 memmove(&rep.pktinfo.v4info, CMSG_DATA(cmsg),
 642                                         sizeof(struct in_pktinfo));
 643                                 break;
 644 #elif defined(IP_RECVDSTADDR)
 645                         } else if( cmsg->cmsg_level == IPPROTO_IP &&
 646                                 cmsg->cmsg_type == IP_RECVDSTADDR) {
 647                                 rep.srctype = 4;
 648                                 memmove(&rep.pktinfo.v4addr, CMSG_DATA(cmsg),
 649                                         sizeof(struct in_addr));
 650                                 break;
 651 #endif /* IP_PKTINFO or IP_RECVDSTADDR */
 652                         }
 653                 }
 654                 if(verbosity >= VERB_ALGO)
 655                         p_ancil("receive_udp on interface", &rep);
 656 #endif /* S_SPLINT_S */
 657                 fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
 658                 if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
 659                         /* send back immediate reply */
 660                         (void)comm_point_send_udp_msg_if(rep.c, rep.c->buffer,
 661                                 (struct sockaddr*)&rep.addr, rep.addrlen, &rep);
 662                 }
 663                 if(!rep.c || rep.c->fd == -1) /* commpoint closed */
 664                         break;
 665         }
 666 #else
 667         (void)fd;
 668         (void)event;
 669         (void)arg;
 670         fatal_exit("recvmsg: No support for IPV6_PKTINFO; IP_PKTINFO or IP_RECVDSTADDR. "
 671                 "Please disable interface-automatic");
 672 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG */
 673 }
 674
 675 void
 676 comm_point_udp_callback(int fd, short event, void* arg)
 677 {
 678         struct comm_reply rep;
 679         ssize_t rcv;
 680         int i;
 681         struct sldns_buffer *buffer;
 682
 683         rep.c = (struct comm_point*)arg;
 684         log_assert(rep.c->type == comm_udp);
 685
 686         if(!(event&UB_EV_READ))
 687                 return;
 688         log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
 689         ub_comm_base_now(rep.c->ev->base);
 690         for(i=0; i<NUM_UDP_PER_SELECT; i++) {
 691                 sldns_buffer_clear(rep.c->buffer);
 692                 rep.addrlen = (socklen_t)sizeof(rep.addr);
 693                 log_assert(fd != -1);
 694                 log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
 695                 rcv = recvfrom(fd, (void*)sldns_buffer_begin(rep.c->buffer),
 696                         sldns_buffer_remaining(rep.c->buffer), 0,
 697                         (struct sockaddr*)&rep.addr, &rep.addrlen);
 698                 if(rcv == -1) {
 699 #ifndef USE_WINSOCK
 700                         if(errno != EAGAIN && errno != EINTR)
 701                                 log_err("recvfrom %d failed: %s",
 702                                         fd, strerror(errno));
 703 #else
 704                         if(WSAGetLastError() != WSAEINPROGRESS &&
 705                                 WSAGetLastError() != WSAECONNRESET &&
 706                                 WSAGetLastError()!= WSAEWOULDBLOCK)
 707                                 log_err("recvfrom failed: %s",
 708                                         wsa_strerror(WSAGetLastError()));
 709 #endif
 710                         return;
 711                 }
 712                 sldns_buffer_skip(rep.c->buffer, rcv);
 713                 sldns_buffer_flip(rep.c->buffer);
 714                 rep.srctype = 0;
 715                 fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
 716                 if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
 717                         /* send back immediate reply */
 718 #ifdef USE_DNSCRYPT
 719                         buffer = rep.c->dnscrypt_buffer;
 720 #else
 721                         buffer = rep.c->buffer;
 722 #endif
 723                         (void)comm_point_send_udp_msg(rep.c, buffer,
 724                                 (struct sockaddr*)&rep.addr, rep.addrlen);
 725                 }
 726                 if(!rep.c || rep.c->fd != fd) /* commpoint closed to -1 or reused for
 727                 another UDP port. Note rep.c cannot be reused with TCP fd. */
 728                         break;
 729         }
 730 }
 731
 732 /** Use a new tcp handler for new query fd, set to read query */
 733 static void
 734 setup_tcp_handler(struct comm_point* c, int fd, int cur, int max)
 735 {
 736         int handler_usage;
 737         log_assert(c->type == comm_tcp || c->type == comm_http);
 738         log_assert(c->fd == -1);
 739         sldns_buffer_clear(c->buffer);
 740 #ifdef USE_DNSCRYPT
 741         if (c->dnscrypt)
 742                 sldns_buffer_clear(c->dnscrypt_buffer);
 743 #endif
 744         c->tcp_is_reading = 1;
 745         c->tcp_byte_count = 0;
 746         /* if more than half the tcp handlers are in use, use a shorter
 747          * timeout for this TCP connection, we need to make space for
 748          * other connections to be able to get attention */
 749         /* If > 50% TCP handler structures in use, set timeout to 1/100th
 750          *      configured value.
 751          * If > 65%TCP handler structures in use, set to 1/500th configured
 752          *      value.
 753          * If > 80% TCP handler structures in use, set to 0.
 754          *
 755          * If the timeout to use falls below 200 milliseconds, an actual
 756          * timeout of 200ms is used.
 757          */
 758         handler_usage = (cur * 100) / max;
 759         if(handler_usage > 50 && handler_usage <= 65)
 760                 c->tcp_timeout_msec /= 100;
 761         else if (handler_usage > 65 && handler_usage <= 80)
 762                 c->tcp_timeout_msec /= 500;
 763         else if (handler_usage > 80)
 764                 c->tcp_timeout_msec = 0;
 765         comm_point_start_listening(c, fd,
 766                 c->tcp_timeout_msec < TCP_QUERY_TIMEOUT_MINIMUM
 767                         ? TCP_QUERY_TIMEOUT_MINIMUM
 768                         : c->tcp_timeout_msec);
 769 }
 770
 771 void comm_base_handle_slow_accept(int ATTR_UNUSED(fd),
 772         short ATTR_UNUSED(event), void* arg)
 773 {
 774         struct comm_base* b = (struct comm_base*)arg;
 775         /* timeout for the slow accept, re-enable accepts again */
 776         if(b->start_accept) {
 777                 verbose(VERB_ALGO, "wait is over, slow accept disabled");
 778                 fptr_ok(fptr_whitelist_start_accept(b->start_accept));
 779                 (*b->start_accept)(b->cb_arg);
 780                 b->eb->slow_accept_enabled = 0;
 781         }
 782 }
 783
 784 int comm_point_perform_accept(struct comm_point* c,
 785         struct sockaddr_storage* addr, socklen_t* addrlen)
 786 {
 787         int new_fd;
 788         *addrlen = (socklen_t)sizeof(*addr);
 789 #ifndef HAVE_ACCEPT4
 790         new_fd = accept(c->fd, (struct sockaddr*)addr, addrlen);
 791 #else
 792         /* SOCK_NONBLOCK saves extra calls to fcntl for the same result */
 793         new_fd = accept4(c->fd, (struct sockaddr*)addr, addrlen, SOCK_NONBLOCK);
 794 #endif
 795         if(new_fd == -1) {
 796 #ifndef USE_WINSOCK
 797                 /* EINTR is signal interrupt. others are closed connection. */
 798                 if(     errno == EINTR || errno == EAGAIN
 799 #ifdef EWOULDBLOCK
 800                         || errno == EWOULDBLOCK
 801 #endif
 802 #ifdef ECONNABORTED
 803                         || errno == ECONNABORTED
 804 #endif
 805 #ifdef EPROTO
 806                         || errno == EPROTO
 807 #endif /* EPROTO */
 808                         )
 809                         return -1;
 810 #if defined(ENFILE) && defined(EMFILE)
 811                 if(errno == ENFILE || errno == EMFILE) {
 812                         /* out of file descriptors, likely outside of our
 813                          * control. stop accept() calls for some time */
 814                         if(c->ev->base->stop_accept) {
 815                                 struct comm_base* b = c->ev->base;
 816                                 struct timeval tv;
 817                                 verbose(VERB_ALGO, "out of file descriptors: "
 818                                         "slow accept");
 819                                 b->eb->slow_accept_enabled = 1;
 820                                 fptr_ok(fptr_whitelist_stop_accept(
 821                                         b->stop_accept));
 822                                 (*b->stop_accept)(b->cb_arg);
 823                                 /* set timeout, no mallocs */
 824                                 tv.tv_sec = NETEVENT_SLOW_ACCEPT_TIME/1000;
 825                                 tv.tv_usec = (NETEVENT_SLOW_ACCEPT_TIME%1000)*1000;
 826                                 b->eb->slow_accept = ub_event_new(b->eb->base,
 827                                         -1, UB_EV_TIMEOUT,
 828                                         comm_base_handle_slow_accept, b);
 829                                 if(b->eb->slow_accept == NULL) {
 830                                         /* we do not want to log here, because
 831                                          * that would spam the logfiles.
 832                                          * error: "event_base_set failed." */
 833                                 }
 834                                 else if(ub_event_add(b->eb->slow_accept, &tv)
 835                                         != 0) {
 836                                         /* we do not want to log here,
 837                                          * error: "event_add failed." */
 838                                 }
 839                         }
 840                         return -1;
 841                 }
 842 #endif
 843 #else /* USE_WINSOCK */
 844                 if(WSAGetLastError() == WSAEINPROGRESS ||
 845                         WSAGetLastError() == WSAECONNRESET)
 846                         return -1;
 847                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
 848                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
 849                         return -1;
 850                 }
 851 #endif
 852                 log_err_addr("accept failed", sock_strerror(errno), addr,
 853                         *addrlen);
 854                 return -1;
 855         }
 856         if(c->tcp_conn_limit && c->type == comm_tcp_accept) {
 857                 c->tcl_addr = tcl_addr_lookup(c->tcp_conn_limit, addr, *addrlen);
 858                 if(!tcl_new_connection(c->tcl_addr)) {
 859                         if(verbosity >= 3)
 860                                 log_err_addr("accept rejected",
 861                                 "connection limit exceeded", addr, *addrlen);
 862                         close(new_fd);
 863                         return -1;
 864                 }
 865         }
 866 #ifndef HAVE_ACCEPT4
 867         fd_set_nonblock(new_fd);
 868 #endif
 869         return new_fd;
 870 }
 871
 872 #ifdef USE_WINSOCK
 873 static long win_bio_cb(BIO *b, int oper, const char* ATTR_UNUSED(argp),
 874         int ATTR_UNUSED(argi), long argl, long retvalue)
 875 {
 876         int wsa_err = WSAGetLastError(); /* store errcode before it is gone */
 877         verbose(VERB_ALGO, "bio_cb %d, %s %s %s", oper,
 878                 (oper&BIO_CB_RETURN)?"return":"before",
 879                 (oper&BIO_CB_READ)?"read":((oper&BIO_CB_WRITE)?"write":"other"),
 880                 wsa_err==WSAEWOULDBLOCK?"wsawb":"");
 881         /* on windows, check if previous operation caused EWOULDBLOCK */
 882         if( (oper == (BIO_CB_READ|BIO_CB_RETURN) && argl == 0) ||
 883                 (oper == (BIO_CB_GETS|BIO_CB_RETURN) && argl == 0)) {
 884                 if(wsa_err == WSAEWOULDBLOCK)
 885                         ub_winsock_tcp_wouldblock((struct ub_event*)
 886                                 BIO_get_callback_arg(b), UB_EV_READ);
 887         }
 888         if( (oper == (BIO_CB_WRITE|BIO_CB_RETURN) && argl == 0) ||
 889                 (oper == (BIO_CB_PUTS|BIO_CB_RETURN) && argl == 0)) {
 890                 if(wsa_err == WSAEWOULDBLOCK)
 891                         ub_winsock_tcp_wouldblock((struct ub_event*)
 892                                 BIO_get_callback_arg(b), UB_EV_WRITE);
 893         }
 894         /* return original return value */
 895         return retvalue;
 896 }
 897
 898 /** set win bio callbacks for nonblocking operations */
 899 void
 900 comm_point_tcp_win_bio_cb(struct comm_point* c, void* thessl)
 901 {
 902         SSL* ssl = (SSL*)thessl;
 903         /* set them both just in case, but usually they are the same BIO */
 904         BIO_set_callback(SSL_get_rbio(ssl), &win_bio_cb);
 905         BIO_set_callback_arg(SSL_get_rbio(ssl), (char*)c->ev->ev);
 906         BIO_set_callback(SSL_get_wbio(ssl), &win_bio_cb);
 907         BIO_set_callback_arg(SSL_get_wbio(ssl), (char*)c->ev->ev);
 908 }
 909 #endif
 910
 911 #ifdef HAVE_NGHTTP2
 912 /** Create http2 session server.  Per connection, after TCP accepted.*/
 913 static int http2_session_server_create(struct http2_session* h2_session)
 914 {
 915         log_assert(h2_session->callbacks);
 916         h2_session->is_drop = 0;
 917         if(nghttp2_session_server_new(&h2_session->session,
 918                         h2_session->callbacks,
 919                 h2_session) == NGHTTP2_ERR_NOMEM) {
 920                 log_err("failed to create nghttp2 session server");
 921                 return 0;
 922         }
 923
 924         return 1;
 925 }
 926
 927 /** Submit http2 setting to session. Once per session. */
 928 static int http2_submit_settings(struct http2_session* h2_session)
 929 {
 930         int ret;
 931         nghttp2_settings_entry settings[1] = {
 932                 {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS,
 933                  h2_session->c->http2_max_streams}};
 934
 935         ret = nghttp2_submit_settings(h2_session->session, NGHTTP2_FLAG_NONE,
 936                 settings, 1);
 937         if(ret) {
 938                 verbose(VERB_QUERY, "http2: submit_settings failed, "
 939                         "error: %s", nghttp2_strerror(ret));
 940                 return 0;
 941         }
 942         return 1;
 943 }
 944 #endif /* HAVE_NGHTTP2 */
 945
 946
 947 void
 948 comm_point_tcp_accept_callback(int fd, short event, void* arg)
 949 {
 950         struct comm_point* c = (struct comm_point*)arg, *c_hdl;
 951         int new_fd;
 952         log_assert(c->type == comm_tcp_accept);
 953         if(!(event & UB_EV_READ)) {
 954                 log_info("ignoring tcp accept event %d", (int)event);
 955                 return;
 956         }
 957         ub_comm_base_now(c->ev->base);
 958         /* find free tcp handler. */
 959         if(!c->tcp_free) {
 960                 log_warn("accepted too many tcp, connections full");
 961                 return;
 962         }
 963         /* accept incoming connection. */
 964         c_hdl = c->tcp_free;
 965         /* clear leftover flags from previous use, and then set the
 966          * correct event base for the event structure for libevent */
 967         ub_event_free(c_hdl->ev->ev);
 968
 969         if(c_hdl->type == comm_http) {
 970 #ifdef HAVE_NGHTTP2
 971                 if(!c_hdl->h2_session ||
 972                         !http2_session_server_create(c_hdl->h2_session)) {
 973                         log_warn("failed to create nghttp2");
 974                         return;
 975                 }
 976                 if(!c_hdl->h2_session ||
 977                         !http2_submit_settings(c_hdl->h2_session)) {
 978                         log_warn("failed to submit http2 settings");
 979                         return;
 980                 }
 981 #endif
 982                 c_hdl->ev->ev = ub_event_new(c_hdl->ev->base->eb->base, -1,
 983                         UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT,
 984                         comm_point_http_handle_callback, c_hdl);
 985         } else {
 986                 c_hdl->ev->ev = ub_event_new(c_hdl->ev->base->eb->base, -1,
 987                         UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT,
 988                         comm_point_tcp_handle_callback, c_hdl);
 989         }
 990         if(!c_hdl->ev->ev) {
 991                 log_warn("could not ub_event_new, dropped tcp");
 992                 return;
 993         }
 994         log_assert(fd != -1);
 995         (void)fd;
 996         new_fd = comm_point_perform_accept(c, &c_hdl->repinfo.addr,
 997                 &c_hdl->repinfo.addrlen);
 998         if(new_fd == -1)
 999                 return;
1000         if(c->ssl) {
1001                 c_hdl->ssl = incoming_ssl_fd(c->ssl, new_fd);
1002                 if(!c_hdl->ssl) {
1003                         c_hdl->fd = new_fd;
1004                         comm_point_close(c_hdl);
1005                         return;
1006                 }
1007                 c_hdl->ssl_shake_state = comm_ssl_shake_read;
1008 #ifdef USE_WINSOCK
1009                 comm_point_tcp_win_bio_cb(c_hdl, c_hdl->ssl);
1010 #endif
1011         }
1012
1013         /* grab the tcp handler buffers */
1014         c->cur_tcp_count++;
1015         c->tcp_free = c_hdl->tcp_free;
1016         if(!c->tcp_free) {
1017                 /* stop accepting incoming queries for now. */
1018                 comm_point_stop_listening(c);
1019         }
1020         setup_tcp_handler(c_hdl, new_fd, c->cur_tcp_count, c->max_tcp_count);
1021 }
1022
1023 /** Make tcp handler free for next assignment */
1024 static void
1025 reclaim_tcp_handler(struct comm_point* c)
1026 {
1027         log_assert(c->type == comm_tcp);
1028         if(c->ssl) {
1029 #ifdef HAVE_SSL
1030                 SSL_shutdown(c->ssl);
1031                 SSL_free(c->ssl);
1032                 c->ssl = NULL;
1033 #endif
1034         }
1035         comm_point_close(c);
1036         if(c->tcp_parent) {
1037                 c->tcp_parent->cur_tcp_count--;
1038                 c->tcp_free = c->tcp_parent->tcp_free;
1039                 c->tcp_parent->tcp_free = c;
1040                 if(!c->tcp_free) {
1041                         /* re-enable listening on accept socket */
1042                         comm_point_start_listening(c->tcp_parent, -1, -1);
1043                 }
1044         }
1045 }
1046
1047 /** do the callback when writing is done */
1048 static void
1049 tcp_callback_writer(struct comm_point* c)
1050 {
1051         log_assert(c->type == comm_tcp);
1052         sldns_buffer_clear(c->buffer);
1053         if(c->tcp_do_toggle_rw)
1054                 c->tcp_is_reading = 1;
1055         c->tcp_byte_count = 0;
1056         /* switch from listening(write) to listening(read) */
1057         if(c->tcp_req_info) {
1058                 tcp_req_info_handle_writedone(c->tcp_req_info);
1059         } else {
1060                 comm_point_stop_listening(c);
1061                 comm_point_start_listening(c, -1, c->tcp_timeout_msec);
1062         }
1063 }
1064
1065 /** do the callback when reading is done */
1066 static void
1067 tcp_callback_reader(struct comm_point* c)
1068 {
1069         log_assert(c->type == comm_tcp || c->type == comm_local);
1070         sldns_buffer_flip(c->buffer);
1071         if(c->tcp_do_toggle_rw)
1072                 c->tcp_is_reading = 0;
1073         c->tcp_byte_count = 0;
1074         if(c->tcp_req_info) {
1075                 tcp_req_info_handle_readdone(c->tcp_req_info);
1076         } else {
1077                 if(c->type == comm_tcp)
1078                         comm_point_stop_listening(c);
1079                 fptr_ok(fptr_whitelist_comm_point(c->callback));
1080                 if( (*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &c->repinfo) ) {
1081                         comm_point_start_listening(c, -1, c->tcp_timeout_msec);
1082                 }
1083         }
1084 }
1085
1086 #ifdef HAVE_SSL
1087 /** true if the ssl handshake error has to be squelched from the logs */
1088 int
1089 squelch_err_ssl_handshake(unsigned long err)
1090 {
1091         if(verbosity >= VERB_QUERY)
1092                 return 0; /* only squelch on low verbosity */
1093         /* this is very specific, we could filter on ERR_GET_REASON()
1094          * (the third element in ERR_PACK) */
1095         if(err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_HTTPS_PROXY_REQUEST) ||
1096                 err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_HTTP_REQUEST) ||
1097                 err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_GET_RECORD, SSL_R_WRONG_VERSION_NUMBER) ||
1098                 err == ERR_PACK(ERR_LIB_SSL, SSL_F_SSL3_READ_BYTES, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE)
1099 #ifdef SSL_F_TLS_POST_PROCESS_CLIENT_HELLO
1100                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_POST_PROCESS_CLIENT_HELLO, SSL_R_NO_SHARED_CIPHER)
1101 #endif
1102 #ifdef SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO
1103                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_UNKNOWN_PROTOCOL)
1104                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_UNSUPPORTED_PROTOCOL)
1105 #  ifdef SSL_R_VERSION_TOO_LOW
1106                 || err == ERR_PACK(ERR_LIB_SSL, SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO, SSL_R_VERSION_TOO_LOW)
1107 #  endif
1108 #endif
1109                 )
1110                 return 1;
1111         return 0;
1112 }
1113 #endif /* HAVE_SSL */
1114
1115 /** continue ssl handshake */
1116 #ifdef HAVE_SSL
1117 static int
1118 ssl_handshake(struct comm_point* c)
1119 {
1120         int r;
1121         if(c->ssl_shake_state == comm_ssl_shake_hs_read) {
1122                 /* read condition satisfied back to writing */
1123                 comm_point_listen_for_rw(c, 1, 1);
1124                 c->ssl_shake_state = comm_ssl_shake_none;
1125                 return 1;
1126         }
1127         if(c->ssl_shake_state == comm_ssl_shake_hs_write) {
1128                 /* write condition satisfied, back to reading */
1129                 comm_point_listen_for_rw(c, 1, 0);
1130                 c->ssl_shake_state = comm_ssl_shake_none;
1131                 return 1;
1132         }
1133
1134         ERR_clear_error();
1135         r = SSL_do_handshake(c->ssl);
1136         if(r != 1) {
1137                 int want = SSL_get_error(c->ssl, r);
1138                 if(want == SSL_ERROR_WANT_READ) {
1139                         if(c->ssl_shake_state == comm_ssl_shake_read)
1140                                 return 1;
1141                         c->ssl_shake_state = comm_ssl_shake_read;
1142                         comm_point_listen_for_rw(c, 1, 0);
1143                         return 1;
1144                 } else if(want == SSL_ERROR_WANT_WRITE) {
1145                         if(c->ssl_shake_state == comm_ssl_shake_write)
1146                                 return 1;
1147                         c->ssl_shake_state = comm_ssl_shake_write;
1148                         comm_point_listen_for_rw(c, 0, 1);
1149                         return 1;
1150                 } else if(r == 0) {
1151                         return 0; /* closed */
1152                 } else if(want == SSL_ERROR_SYSCALL) {
1153                         /* SYSCALL and errno==0 means closed uncleanly */
1154 #ifdef EPIPE
1155                         if(errno == EPIPE && verbosity < 2)
1156                                 return 0; /* silence 'broken pipe' */
1157 #endif
1158 #ifdef ECONNRESET
1159                         if(errno == ECONNRESET && verbosity < 2)
1160                                 return 0; /* silence reset by peer */
1161 #endif
1162                         if(errno != 0)
1163                                 log_err("SSL_handshake syscall: %s",
1164                                         strerror(errno));
1165                         return 0;
1166                 } else {
1167                         unsigned long err = ERR_get_error();
1168                         if(!squelch_err_ssl_handshake(err)) {
1169                                 log_crypto_err_code("ssl handshake failed", err);
1170                                 log_addr(VERB_OPS, "ssl handshake failed", &c->repinfo.addr,
1171                                         c->repinfo.addrlen);
1172                         }
1173                         return 0;
1174                 }
1175         }
1176         /* this is where peer verification could take place */
1177         if((SSL_get_verify_mode(c->ssl)&SSL_VERIFY_PEER)) {
1178                 /* verification */
1179                 if(SSL_get_verify_result(c->ssl) == X509_V_OK) {
1180                         X509* x = SSL_get_peer_certificate(c->ssl);
1181                         if(!x) {
1182                                 log_addr(VERB_ALGO, "SSL connection failed: "
1183                                         "no certificate",
1184                                         &c->repinfo.addr, c->repinfo.addrlen);
1185                                 return 0;
1186                         }
1187                         log_cert(VERB_ALGO, "peer certificate", x);
1188 #ifdef HAVE_SSL_GET0_PEERNAME
1189                         if(SSL_get0_peername(c->ssl)) {
1190                                 char buf[255];
1191                                 snprintf(buf, sizeof(buf), "SSL connection "
1192                                         "to %s authenticated",
1193                                         SSL_get0_peername(c->ssl));
1194                                 log_addr(VERB_ALGO, buf, &c->repinfo.addr,
1195                                         c->repinfo.addrlen);
1196                         } else {
1197 #endif
1198                                 log_addr(VERB_ALGO, "SSL connection "
1199                                         "authenticated", &c->repinfo.addr,
1200                                         c->repinfo.addrlen);
1201 #ifdef HAVE_SSL_GET0_PEERNAME
1202                         }
1203 #endif
1204                         X509_free(x);
1205                 } else {
1206                         X509* x = SSL_get_peer_certificate(c->ssl);
1207                         if(x) {
1208                                 log_cert(VERB_ALGO, "peer certificate", x);
1209                                 X509_free(x);
1210                         }
1211                         log_addr(VERB_ALGO, "SSL connection failed: "
1212                                 "failed to authenticate",
1213                                 &c->repinfo.addr, c->repinfo.addrlen);
1214                         return 0;
1215                 }
1216         } else {
1217                 /* unauthenticated, the verify peer flag was not set
1218                  * in c->ssl when the ssl object was created from ssl_ctx */
1219                 log_addr(VERB_ALGO, "SSL connection", &c->repinfo.addr,
1220                         c->repinfo.addrlen);
1221         }
1222
1223         /* check if http2 use is negotiated */
1224         if(c->type == comm_http && c->h2_session) {
1225                 const unsigned char *alpn;
1226                 unsigned int alpnlen = 0;
1227                 SSL_get0_alpn_selected(c->ssl, &alpn, &alpnlen);
1228                 if(alpnlen == 2 && memcmp("h2", alpn, 2) == 0) {
1229                         /* connection upgraded to HTTP2 */
1230                         c->tcp_do_toggle_rw = 0;
1231                         c->use_h2 = 1;
1232                 }
1233         }
1234
1235         /* setup listen rw correctly */
1236         if(c->tcp_is_reading) {
1237                 if(c->ssl_shake_state != comm_ssl_shake_read)
1238                         comm_point_listen_for_rw(c, 1, 0);
1239         } else {
1240                 comm_point_listen_for_rw(c, 1, 1);
1241         }
1242         c->ssl_shake_state = comm_ssl_shake_none;
1243         return 1;
1244 }
1245 #endif /* HAVE_SSL */
1246
1247 /** ssl read callback on TCP */
1248 static int
1249 ssl_handle_read(struct comm_point* c)
1250 {
1251 #ifdef HAVE_SSL
1252         int r;
1253         if(c->ssl_shake_state != comm_ssl_shake_none) {
1254                 if(!ssl_handshake(c))
1255                         return 0;
1256                 if(c->ssl_shake_state != comm_ssl_shake_none)
1257                         return 1;
1258         }
1259         if(c->tcp_byte_count < sizeof(uint16_t)) {
1260                 /* read length bytes */
1261                 ERR_clear_error();
1262                 if((r=SSL_read(c->ssl, (void*)sldns_buffer_at(c->buffer,
1263                         c->tcp_byte_count), (int)(sizeof(uint16_t) -
1264                         c->tcp_byte_count))) <= 0) {
1265                         int want = SSL_get_error(c->ssl, r);
1266                         if(want == SSL_ERROR_ZERO_RETURN) {
1267                                 if(c->tcp_req_info)
1268                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1269                                 return 0; /* shutdown, closed */
1270                         } else if(want == SSL_ERROR_WANT_READ) {
1271                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1272                                 return 1; /* read more later */
1273                         } else if(want == SSL_ERROR_WANT_WRITE) {
1274                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1275                                 comm_point_listen_for_rw(c, 0, 1);
1276                                 return 1;
1277                         } else if(want == SSL_ERROR_SYSCALL) {
1278 #ifdef ECONNRESET
1279                                 if(errno == ECONNRESET && verbosity < 2)
1280                                         return 0; /* silence reset by peer */
1281 #endif
1282                                 if(errno != 0)
1283                                         log_err("SSL_read syscall: %s",
1284                                                 strerror(errno));
1285                                 return 0;
1286                         }
1287                         log_crypto_err("could not SSL_read");
1288                         return 0;
1289                 }
1290                 c->tcp_byte_count += r;
1291                 if(c->tcp_byte_count < sizeof(uint16_t))
1292                         return 1;
1293                 if(sldns_buffer_read_u16_at(c->buffer, 0) >
1294                         sldns_buffer_capacity(c->buffer)) {
1295                         verbose(VERB_QUERY, "ssl: dropped larger than buffer");
1296                         return 0;
1297                 }
1298                 sldns_buffer_set_limit(c->buffer,
1299                         sldns_buffer_read_u16_at(c->buffer, 0));
1300                 if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1301                         verbose(VERB_QUERY, "ssl: dropped bogus too short.");
1302                         return 0;
1303                 }
1304                 sldns_buffer_skip(c->buffer, (ssize_t)(c->tcp_byte_count-sizeof(uint16_t)));
1305                 verbose(VERB_ALGO, "Reading ssl tcp query of length %d",
1306                         (int)sldns_buffer_limit(c->buffer));
1307         }
1308         if(sldns_buffer_remaining(c->buffer) > 0) {
1309                 ERR_clear_error();
1310                 r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1311                         (int)sldns_buffer_remaining(c->buffer));
1312                 if(r <= 0) {
1313                         int want = SSL_get_error(c->ssl, r);
1314                         if(want == SSL_ERROR_ZERO_RETURN) {
1315                                 if(c->tcp_req_info)
1316                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1317                                 return 0; /* shutdown, closed */
1318                         } else if(want == SSL_ERROR_WANT_READ) {
1319                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1320                                 return 1; /* read more later */
1321                         } else if(want == SSL_ERROR_WANT_WRITE) {
1322                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1323                                 comm_point_listen_for_rw(c, 0, 1);
1324                                 return 1;
1325                         } else if(want == SSL_ERROR_SYSCALL) {
1326 #ifdef ECONNRESET
1327                                 if(errno == ECONNRESET && verbosity < 2)
1328                                         return 0; /* silence reset by peer */
1329 #endif
1330                                 if(errno != 0)
1331                                         log_err("SSL_read syscall: %s",
1332                                                 strerror(errno));
1333                                 return 0;
1334                         }
1335                         log_crypto_err("could not SSL_read");
1336                         return 0;
1337                 }
1338                 sldns_buffer_skip(c->buffer, (ssize_t)r);
1339         }
1340         if(sldns_buffer_remaining(c->buffer) <= 0) {
1341                 tcp_callback_reader(c);
1342         }
1343         return 1;
1344 #else
1345         (void)c;
1346         return 0;
1347 #endif /* HAVE_SSL */
1348 }
1349
1350 /** ssl write callback on TCP */
1351 static int
1352 ssl_handle_write(struct comm_point* c)
1353 {
1354 #ifdef HAVE_SSL
1355         int r;
1356         if(c->ssl_shake_state != comm_ssl_shake_none) {
1357                 if(!ssl_handshake(c))
1358                         return 0;
1359                 if(c->ssl_shake_state != comm_ssl_shake_none)
1360                         return 1;
1361         }
1362         /* ignore return, if fails we may simply block */
1363         (void)SSL_set_mode(c->ssl, (long)SSL_MODE_ENABLE_PARTIAL_WRITE);
1364         if(c->tcp_byte_count < sizeof(uint16_t)) {
1365                 uint16_t len = htons(sldns_buffer_limit(c->buffer));
1366                 ERR_clear_error();
1367                 if(sizeof(uint16_t)+sldns_buffer_remaining(c->buffer) <
1368                         LDNS_RR_BUF_SIZE) {
1369                         /* combine the tcp length and the query for write,
1370                          * this emulates writev */
1371                         uint8_t buf[LDNS_RR_BUF_SIZE];
1372                         memmove(buf, &len, sizeof(uint16_t));
1373                         memmove(buf+sizeof(uint16_t),
1374                                 sldns_buffer_current(c->buffer),
1375                                 sldns_buffer_remaining(c->buffer));
1376                         r = SSL_write(c->ssl, (void*)(buf+c->tcp_byte_count),
1377                                 (int)(sizeof(uint16_t)+
1378                                 sldns_buffer_remaining(c->buffer)
1379                                 - c->tcp_byte_count));
1380                 } else {
1381                         r = SSL_write(c->ssl,
1382                                 (void*)(((uint8_t*)&len)+c->tcp_byte_count),
1383                                 (int)(sizeof(uint16_t)-c->tcp_byte_count));
1384                 }
1385                 if(r <= 0) {
1386                         int want = SSL_get_error(c->ssl, r);
1387                         if(want == SSL_ERROR_ZERO_RETURN) {
1388                                 return 0; /* closed */
1389                         } else if(want == SSL_ERROR_WANT_READ) {
1390                                 c->ssl_shake_state = comm_ssl_shake_hs_read;
1391                                 comm_point_listen_for_rw(c, 1, 0);
1392                                 return 1; /* wait for read condition */
1393                         } else if(want == SSL_ERROR_WANT_WRITE) {
1394                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1395                                 return 1; /* write more later */
1396                         } else if(want == SSL_ERROR_SYSCALL) {
1397 #ifdef EPIPE
1398                                 if(errno == EPIPE && verbosity < 2)
1399                                         return 0; /* silence 'broken pipe' */
1400 #endif
1401                                 if(errno != 0)
1402                                         log_err("SSL_write syscall: %s",
1403                                                 strerror(errno));
1404                                 return 0;
1405                         }
1406                         log_crypto_err("could not SSL_write");
1407                         return 0;
1408                 }
1409                 c->tcp_byte_count += r;
1410                 if(c->tcp_byte_count < sizeof(uint16_t))
1411                         return 1;
1412                 sldns_buffer_set_position(c->buffer, c->tcp_byte_count -
1413                         sizeof(uint16_t));
1414                 if(sldns_buffer_remaining(c->buffer) == 0) {
1415                         tcp_callback_writer(c);
1416                         return 1;
1417                 }
1418         }
1419         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1420         ERR_clear_error();
1421         r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
1422                 (int)sldns_buffer_remaining(c->buffer));
1423         if(r <= 0) {
1424                 int want = SSL_get_error(c->ssl, r);
1425                 if(want == SSL_ERROR_ZERO_RETURN) {
1426                         return 0; /* closed */
1427                 } else if(want == SSL_ERROR_WANT_READ) {
1428                         c->ssl_shake_state = comm_ssl_shake_hs_read;
1429                         comm_point_listen_for_rw(c, 1, 0);
1430                         return 1; /* wait for read condition */
1431                 } else if(want == SSL_ERROR_WANT_WRITE) {
1432                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1433                         return 1; /* write more later */
1434                 } else if(want == SSL_ERROR_SYSCALL) {
1435 #ifdef EPIPE
1436                         if(errno == EPIPE && verbosity < 2)
1437                                 return 0; /* silence 'broken pipe' */
1438 #endif
1439                         if(errno != 0)
1440                                 log_err("SSL_write syscall: %s",
1441                                         strerror(errno));
1442                         return 0;
1443                 }
1444                 log_crypto_err("could not SSL_write");
1445                 return 0;
1446         }
1447         sldns_buffer_skip(c->buffer, (ssize_t)r);
1448
1449         if(sldns_buffer_remaining(c->buffer) == 0) {
1450                 tcp_callback_writer(c);
1451         }
1452         return 1;
1453 #else
1454         (void)c;
1455         return 0;
1456 #endif /* HAVE_SSL */
1457 }
1458
1459 /** handle ssl tcp connection with dns contents */
1460 static int
1461 ssl_handle_it(struct comm_point* c)
1462 {
1463         if(c->tcp_is_reading)
1464                 return ssl_handle_read(c);
1465         return ssl_handle_write(c);
1466 }
1467
1468 /** Handle tcp reading callback.
1469  * @param fd: file descriptor of socket.
1470  * @param c: comm point to read from into buffer.
1471  * @param short_ok: if true, very short packets are OK (for comm_local).
1472  * @return: 0 on error
1473  */
1474 static int
1475 comm_point_tcp_handle_read(int fd, struct comm_point* c, int short_ok)
1476 {
1477         ssize_t r;
1478         log_assert(c->type == comm_tcp || c->type == comm_local);
1479         if(c->ssl)
1480                 return ssl_handle_it(c);
1481         if(!c->tcp_is_reading)
1482                 return 0;
1483
1484         log_assert(fd != -1);
1485         if(c->tcp_byte_count < sizeof(uint16_t)) {
1486                 /* read length bytes */
1487                 r = recv(fd,(void*)sldns_buffer_at(c->buffer,c->tcp_byte_count),
1488                         sizeof(uint16_t)-c->tcp_byte_count, 0);
1489                 if(r == 0) {
1490                         if(c->tcp_req_info)
1491                                 return tcp_req_info_handle_read_close(c->tcp_req_info);
1492                         return 0;
1493                 } else if(r == -1) {
1494 #ifndef USE_WINSOCK
1495                         if(errno == EINTR || errno == EAGAIN)
1496                                 return 1;
1497 #ifdef ECONNRESET
1498                         if(errno == ECONNRESET && verbosity < 2)
1499                                 return 0; /* silence reset by peer */
1500 #endif
1501 #else /* USE_WINSOCK */
1502                         if(WSAGetLastError() == WSAECONNRESET)
1503                                 return 0;
1504                         if(WSAGetLastError() == WSAEINPROGRESS)
1505                                 return 1;
1506                         if(WSAGetLastError() == WSAEWOULDBLOCK) {
1507                                 ub_winsock_tcp_wouldblock(c->ev->ev,
1508                                         UB_EV_READ);
1509                                 return 1;
1510                         }
1511 #endif
1512                         log_err_addr("read (in tcp s)", sock_strerror(errno),
1513                                 &c->repinfo.addr, c->repinfo.addrlen);
1514                         return 0;
1515                 }
1516                 c->tcp_byte_count += r;
1517                 if(c->tcp_byte_count != sizeof(uint16_t))
1518                         return 1;
1519                 if(sldns_buffer_read_u16_at(c->buffer, 0) >
1520                         sldns_buffer_capacity(c->buffer)) {
1521                         verbose(VERB_QUERY, "tcp: dropped larger than buffer");
1522                         return 0;
1523                 }
1524                 sldns_buffer_set_limit(c->buffer,
1525                         sldns_buffer_read_u16_at(c->buffer, 0));
1526                 if(!short_ok &&
1527                         sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1528                         verbose(VERB_QUERY, "tcp: dropped bogus too short.");
1529                         return 0;
1530                 }
1531                 verbose(VERB_ALGO, "Reading tcp query of length %d",
1532                         (int)sldns_buffer_limit(c->buffer));
1533         }
1534
1535         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1536         r = recv(fd, (void*)sldns_buffer_current(c->buffer),
1537                 sldns_buffer_remaining(c->buffer), 0);
1538         if(r == 0) {
1539                 if(c->tcp_req_info)
1540                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1541                 return 0;
1542         } else if(r == -1) {
1543 #ifndef USE_WINSOCK
1544                 if(errno == EINTR || errno == EAGAIN)
1545                         return 1;
1546 #else /* USE_WINSOCK */
1547                 if(WSAGetLastError() == WSAECONNRESET)
1548                         return 0;
1549                 if(WSAGetLastError() == WSAEINPROGRESS)
1550                         return 1;
1551                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1552                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1553                         return 1;
1554                 }
1555 #endif
1556                 log_err_addr("read (in tcp r)", sock_strerror(errno),
1557                         &c->repinfo.addr, c->repinfo.addrlen);
1558                 return 0;
1559         }
1560         sldns_buffer_skip(c->buffer, r);
1561         if(sldns_buffer_remaining(c->buffer) <= 0) {
1562                 tcp_callback_reader(c);
1563         }
1564         return 1;
1565 }
1566
1567 /**
1568  * Handle tcp writing callback.
1569  * @param fd: file descriptor of socket.
1570  * @param c: comm point to write buffer out of.
1571  * @return: 0 on error
1572  */
1573 static int
1574 comm_point_tcp_handle_write(int fd, struct comm_point* c)
1575 {
1576         ssize_t r;
1577         struct sldns_buffer *buffer;
1578         log_assert(c->type == comm_tcp);
1579 #ifdef USE_DNSCRYPT
1580         buffer = c->dnscrypt_buffer;
1581 #else
1582         buffer = c->buffer;
1583 #endif
1584         if(c->tcp_is_reading && !c->ssl)
1585                 return 0;
1586         log_assert(fd != -1);
1587         if(c->tcp_byte_count == 0 && c->tcp_check_nb_connect) {
1588                 /* check for pending error from nonblocking connect */
1589                 /* from Stevens, unix network programming, vol1, 3rd ed, p450*/
1590                 int error = 0;
1591                 socklen_t len = (socklen_t)sizeof(error);
1592                 if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
1593                         &len) < 0){
1594 #ifndef USE_WINSOCK
1595                         error = errno; /* on solaris errno is error */
1596 #else /* USE_WINSOCK */
1597                         error = WSAGetLastError();
1598 #endif
1599                 }
1600 #ifndef USE_WINSOCK
1601 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
1602                 if(error == EINPROGRESS || error == EWOULDBLOCK)
1603                         return 1; /* try again later */
1604                 else
1605 #endif
1606                 if(error != 0 && verbosity < 2)
1607                         return 0; /* silence lots of chatter in the logs */
1608                 else if(error != 0) {
1609                         log_err_addr("tcp connect", strerror(error),
1610                                 &c->repinfo.addr, c->repinfo.addrlen);
1611 #else /* USE_WINSOCK */
1612                 /* examine error */
1613                 if(error == WSAEINPROGRESS)
1614                         return 1;
1615                 else if(error == WSAEWOULDBLOCK) {
1616                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1617                         return 1;
1618                 } else if(error != 0 && verbosity < 2)
1619                         return 0;
1620                 else if(error != 0) {
1621                         log_err_addr("tcp connect", wsa_strerror(error),
1622                                 &c->repinfo.addr, c->repinfo.addrlen);
1623 #endif /* USE_WINSOCK */
1624                         return 0;
1625                 }
1626         }
1627         if(c->ssl)
1628                 return ssl_handle_it(c);
1629
1630 #ifdef USE_MSG_FASTOPEN
1631         /* Only try this on first use of a connection that uses tfo,
1632            otherwise fall through to normal write */
1633         /* Also, TFO support on WINDOWS not implemented at the moment */
1634         if(c->tcp_do_fastopen == 1) {
1635                 /* this form of sendmsg() does both a connect() and send() so need to
1636                    look for various flavours of error*/
1637                 uint16_t len = htons(sldns_buffer_limit(buffer));
1638                 struct msghdr msg;
1639                 struct iovec iov[2];
1640                 c->tcp_do_fastopen = 0;
1641                 memset(&msg, 0, sizeof(msg));
1642                 iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
1643                 iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
1644                 iov[1].iov_base = sldns_buffer_begin(buffer);
1645                 iov[1].iov_len = sldns_buffer_limit(buffer);
1646                 log_assert(iov[0].iov_len > 0);
1647                 msg.msg_name = &c->repinfo.addr;
1648                 msg.msg_namelen = c->repinfo.addrlen;
1649                 msg.msg_iov = iov;
1650                 msg.msg_iovlen = 2;
1651                 r = sendmsg(fd, &msg, MSG_FASTOPEN);
1652                 if (r == -1) {
1653 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
1654                         /* Handshake is underway, maybe because no TFO cookie available.
1655                            Come back to write the message*/
1656                         if(errno == EINPROGRESS || errno == EWOULDBLOCK)
1657                                 return 1;
1658 #endif
1659                         if(errno == EINTR || errno == EAGAIN)
1660                                 return 1;
1661                         /* Not handling EISCONN here as shouldn't ever hit that case.*/
1662                         if(errno != EPIPE && errno != 0 && verbosity < 2)
1663                                 return 0; /* silence lots of chatter in the logs */
1664                         if(errno != EPIPE && errno != 0) {
1665                                 log_err_addr("tcp sendmsg", strerror(errno),
1666                                         &c->repinfo.addr, c->repinfo.addrlen);
1667                                 return 0;
1668                         }
1669                         /* fallthrough to nonFASTOPEN
1670                          * (MSG_FASTOPEN on Linux 3 produces EPIPE)
1671                          * we need to perform connect() */
1672                         if(connect(fd, (struct sockaddr *)&c->repinfo.addr, c->repinfo.addrlen) == -1) {
1673 #ifdef EINPROGRESS
1674                                 if(errno == EINPROGRESS)
1675                                         return 1; /* wait until connect done*/
1676 #endif
1677 #ifdef USE_WINSOCK
1678                                 if(WSAGetLastError() == WSAEINPROGRESS ||
1679                                         WSAGetLastError() == WSAEWOULDBLOCK)
1680                                         return 1; /* wait until connect done*/
1681 #endif
1682                                 if(tcp_connect_errno_needs_log(
1683                                         (struct sockaddr *)&c->repinfo.addr, c->repinfo.addrlen)) {
1684                                         log_err_addr("outgoing tcp: connect after EPIPE for fastopen",
1685                                                 strerror(errno), &c->repinfo.addr, c->repinfo.addrlen);
1686                                 }
1687                                 return 0;
1688                         }
1689
1690                 } else {
1691                         c->tcp_byte_count += r;
1692                         if(c->tcp_byte_count < sizeof(uint16_t))
1693                                 return 1;
1694                         sldns_buffer_set_position(buffer, c->tcp_byte_count -
1695                                 sizeof(uint16_t));
1696                         if(sldns_buffer_remaining(buffer) == 0) {
1697                                 tcp_callback_writer(c);
1698                                 return 1;
1699                         }
1700                 }
1701         }
1702 #endif /* USE_MSG_FASTOPEN */
1703
1704         if(c->tcp_byte_count < sizeof(uint16_t)) {
1705                 uint16_t len = htons(sldns_buffer_limit(buffer));
1706 #ifdef HAVE_WRITEV
1707                 struct iovec iov[2];
1708                 iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
1709                 iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
1710                 iov[1].iov_base = sldns_buffer_begin(buffer);
1711                 iov[1].iov_len = sldns_buffer_limit(buffer);
1712                 log_assert(iov[0].iov_len > 0);
1713                 r = writev(fd, iov, 2);
1714 #else /* HAVE_WRITEV */
1715                 r = send(fd, (void*)(((uint8_t*)&len)+c->tcp_byte_count),
1716                         sizeof(uint16_t)-c->tcp_byte_count, 0);
1717 #endif /* HAVE_WRITEV */
1718                 if(r == -1) {
1719 #ifndef USE_WINSOCK
1720 #  ifdef EPIPE
1721                         if(errno == EPIPE && verbosity < 2)
1722                                 return 0; /* silence 'broken pipe' */
1723   #endif
1724                         if(errno == EINTR || errno == EAGAIN)
1725                                 return 1;
1726 #ifdef ECONNRESET
1727                         if(errno == ECONNRESET && verbosity < 2)
1728                                 return 0; /* silence reset by peer */
1729 #endif
1730 #  ifdef HAVE_WRITEV
1731                         log_err_addr("tcp writev", strerror(errno),
1732                                 &c->repinfo.addr, c->repinfo.addrlen);
1733 #  else /* HAVE_WRITEV */
1734                         log_err_addr("tcp send s", strerror(errno),
1735                                 &c->repinfo.addr, c->repinfo.addrlen);
1736 #  endif /* HAVE_WRITEV */
1737 #else
1738                         if(WSAGetLastError() == WSAENOTCONN)
1739                                 return 1;
1740                         if(WSAGetLastError() == WSAEINPROGRESS)
1741                                 return 1;
1742                         if(WSAGetLastError() == WSAEWOULDBLOCK) {
1743                                 ub_winsock_tcp_wouldblock(c->ev->ev,
1744                                         UB_EV_WRITE);
1745                                 return 1;
1746                         }
1747                         if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
1748                                 return 0; /* silence reset by peer */
1749                         log_err_addr("tcp send s",
1750                                 wsa_strerror(WSAGetLastError()),
1751                                 &c->repinfo.addr, c->repinfo.addrlen);
1752 #endif
1753                         return 0;
1754                 }
1755                 c->tcp_byte_count += r;
1756                 if(c->tcp_byte_count < sizeof(uint16_t))
1757                         return 1;
1758                 sldns_buffer_set_position(buffer, c->tcp_byte_count -
1759                         sizeof(uint16_t));
1760                 if(sldns_buffer_remaining(buffer) == 0) {
1761                         tcp_callback_writer(c);
1762                         return 1;
1763                 }
1764         }
1765         log_assert(sldns_buffer_remaining(buffer) > 0);
1766         r = send(fd, (void*)sldns_buffer_current(buffer),
1767                 sldns_buffer_remaining(buffer), 0);
1768         if(r == -1) {
1769 #ifndef USE_WINSOCK
1770                 if(errno == EINTR || errno == EAGAIN)
1771                         return 1;
1772 #ifdef ECONNRESET
1773                 if(errno == ECONNRESET && verbosity < 2)
1774                         return 0; /* silence reset by peer */
1775 #endif
1776 #else
1777                 if(WSAGetLastError() == WSAEINPROGRESS)
1778                         return 1;
1779                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1780                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1781                         return 1;
1782                 }
1783                 if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
1784                         return 0; /* silence reset by peer */
1785 #endif
1786                 log_err_addr("tcp send r", sock_strerror(errno),
1787                         &c->repinfo.addr, c->repinfo.addrlen);
1788                 return 0;
1789         }
1790         sldns_buffer_skip(buffer, r);
1791
1792         if(sldns_buffer_remaining(buffer) == 0) {
1793                 tcp_callback_writer(c);
1794         }
1795
1796         return 1;
1797 }
1798
1799 /** read again to drain buffers when there could be more to read */
1800 static void
1801 tcp_req_info_read_again(int fd, struct comm_point* c)
1802 {
1803         while(c->tcp_req_info->read_again) {
1804                 int r;
1805                 c->tcp_req_info->read_again = 0;
1806                 if(c->tcp_is_reading)
1807                         r = comm_point_tcp_handle_read(fd, c, 0);
1808                 else    r = comm_point_tcp_handle_write(fd, c);
1809                 if(!r) {
1810                         reclaim_tcp_handler(c);
1811                         if(!c->tcp_do_close) {
1812                                 fptr_ok(fptr_whitelist_comm_point(
1813                                         c->callback));
1814                                 (void)(*c->callback)(c, c->cb_arg,
1815                                         NETEVENT_CLOSED, NULL);
1816                         }
1817                         return;
1818                 }
1819         }
1820 }
1821
1822 void
1823 comm_point_tcp_handle_callback(int fd, short event, void* arg)
1824 {
1825         struct comm_point* c = (struct comm_point*)arg;
1826         log_assert(c->type == comm_tcp);
1827         ub_comm_base_now(c->ev->base);
1828
1829 #ifdef USE_DNSCRYPT
1830         /* Initialize if this is a dnscrypt socket */
1831         if(c->tcp_parent) {
1832                 c->dnscrypt = c->tcp_parent->dnscrypt;
1833         }
1834         if(c->dnscrypt && c->dnscrypt_buffer == c->buffer) {
1835                 c->dnscrypt_buffer = sldns_buffer_new(sldns_buffer_capacity(c->buffer));
1836                 if(!c->dnscrypt_buffer) {
1837                         log_err("Could not allocate dnscrypt buffer");
1838                         reclaim_tcp_handler(c);
1839                         if(!c->tcp_do_close) {
1840                                 fptr_ok(fptr_whitelist_comm_point(
1841                                         c->callback));
1842                                 (void)(*c->callback)(c, c->cb_arg,
1843                                         NETEVENT_CLOSED, NULL);
1844                         }
1845                         return;
1846                 }
1847         }
1848 #endif
1849
1850         if(event&UB_EV_TIMEOUT) {
1851                 verbose(VERB_QUERY, "tcp took too long, dropped");
1852                 reclaim_tcp_handler(c);
1853                 if(!c->tcp_do_close) {
1854                         fptr_ok(fptr_whitelist_comm_point(c->callback));
1855                         (void)(*c->callback)(c, c->cb_arg,
1856                                 NETEVENT_TIMEOUT, NULL);
1857                 }
1858                 return;
1859         }
1860         if(event&UB_EV_READ) {
1861                 int has_tcpq = (c->tcp_req_info != NULL);
1862                 if(!comm_point_tcp_handle_read(fd, c, 0)) {
1863                         reclaim_tcp_handler(c);
1864                         if(!c->tcp_do_close) {
1865                                 fptr_ok(fptr_whitelist_comm_point(
1866                                         c->callback));
1867                                 (void)(*c->callback)(c, c->cb_arg,
1868                                         NETEVENT_CLOSED, NULL);
1869                         }
1870                 }
1871                 if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again)
1872                         tcp_req_info_read_again(fd, c);
1873                 return;
1874         }
1875         if(event&UB_EV_WRITE) {
1876                 int has_tcpq = (c->tcp_req_info != NULL);
1877                 if(!comm_point_tcp_handle_write(fd, c)) {
1878                         reclaim_tcp_handler(c);
1879                         if(!c->tcp_do_close) {
1880                                 fptr_ok(fptr_whitelist_comm_point(
1881                                         c->callback));
1882                                 (void)(*c->callback)(c, c->cb_arg,
1883                                         NETEVENT_CLOSED, NULL);
1884                         }
1885                 }
1886                 if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again)
1887                         tcp_req_info_read_again(fd, c);
1888                 return;
1889         }
1890         log_err("Ignored event %d for tcphdl.", event);
1891 }
1892
1893 /** Make http handler free for next assignment */
1894 static void
1895 reclaim_http_handler(struct comm_point* c)
1896 {
1897         log_assert(c->type == comm_http);
1898         if(c->ssl) {
1899 #ifdef HAVE_SSL
1900                 SSL_shutdown(c->ssl);
1901                 SSL_free(c->ssl);
1902                 c->ssl = NULL;
1903 #endif
1904         }
1905         comm_point_close(c);
1906         if(c->tcp_parent) {
1907                 c->tcp_parent->cur_tcp_count--;
1908                 c->tcp_free = c->tcp_parent->tcp_free;
1909                 c->tcp_parent->tcp_free = c;
1910                 if(!c->tcp_free) {
1911                         /* re-enable listening on accept socket */
1912                         comm_point_start_listening(c->tcp_parent, -1, -1);
1913                 }
1914         }
1915 }
1916
1917 /** read more data for http (with ssl) */
1918 static int
1919 ssl_http_read_more(struct comm_point* c)
1920 {
1921 #ifdef HAVE_SSL
1922         int r;
1923         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1924         ERR_clear_error();
1925         r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1926                 (int)sldns_buffer_remaining(c->buffer));
1927         if(r <= 0) {
1928                 int want = SSL_get_error(c->ssl, r);
1929                 if(want == SSL_ERROR_ZERO_RETURN) {
1930                         return 0; /* shutdown, closed */
1931                 } else if(want == SSL_ERROR_WANT_READ) {
1932                         return 1; /* read more later */
1933                 } else if(want == SSL_ERROR_WANT_WRITE) {
1934                         c->ssl_shake_state = comm_ssl_shake_hs_write;
1935                         comm_point_listen_for_rw(c, 0, 1);
1936                         return 1;
1937                 } else if(want == SSL_ERROR_SYSCALL) {
1938 #ifdef ECONNRESET
1939                         if(errno == ECONNRESET && verbosity < 2)
1940                                 return 0; /* silence reset by peer */
1941 #endif
1942                         if(errno != 0)
1943                                 log_err("SSL_read syscall: %s",
1944                                         strerror(errno));
1945                         return 0;
1946                 }
1947                 log_crypto_err("could not SSL_read");
1948                 return 0;
1949         }
1950         sldns_buffer_skip(c->buffer, (ssize_t)r);
1951         return 1;
1952 #else
1953         (void)c;
1954         return 0;
1955 #endif /* HAVE_SSL */
1956 }
1957
1958 /** read more data for http */
1959 static int
1960 http_read_more(int fd, struct comm_point* c)
1961 {
1962         ssize_t r;
1963         log_assert(sldns_buffer_remaining(c->buffer) > 0);
1964         r = recv(fd, (void*)sldns_buffer_current(c->buffer),
1965                 sldns_buffer_remaining(c->buffer), 0);
1966         if(r == 0) {
1967                 return 0;
1968         } else if(r == -1) {
1969 #ifndef USE_WINSOCK
1970                 if(errno == EINTR || errno == EAGAIN)
1971                         return 1;
1972 #else /* USE_WINSOCK */
1973                 if(WSAGetLastError() == WSAECONNRESET)
1974                         return 0;
1975                 if(WSAGetLastError() == WSAEINPROGRESS)
1976                         return 1;
1977                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1978                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1979                         return 1;
1980                 }
1981 #endif
1982                 log_err_addr("read (in http r)", sock_strerror(errno),
1983                         &c->repinfo.addr, c->repinfo.addrlen);
1984                 return 0;
1985         }
1986         sldns_buffer_skip(c->buffer, r);
1987         return 1;
1988 }
1989
1990 /** return true if http header has been read (one line complete) */
1991 static int
1992 http_header_done(sldns_buffer* buf)
1993 {
1994         size_t i;
1995         for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
1996                 /* there was a \r before the \n, but we ignore that */
1997                 if((char)sldns_buffer_read_u8_at(buf, i) == '\n')
1998                         return 1;
1999         }
2000         return 0;
2001 }
2002
2003 /** return character string into buffer for header line, moves buffer
2004  * past that line and puts zero terminator into linefeed-newline */
2005 static char*
2006 http_header_line(sldns_buffer* buf)
2007 {
2008         char* result = (char*)sldns_buffer_current(buf);
2009         size_t i;
2010         for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
2011                 /* terminate the string on the \r */
2012                 if((char)sldns_buffer_read_u8_at(buf, i) == '\r')
2013                         sldns_buffer_write_u8_at(buf, i, 0);
2014                 /* terminate on the \n and skip past the it and done */
2015                 if((char)sldns_buffer_read_u8_at(buf, i) == '\n') {
2016                         sldns_buffer_write_u8_at(buf, i, 0);
2017                         sldns_buffer_set_position(buf, i+1);
2018                         return result;
2019                 }
2020         }
2021         return NULL;
2022 }
2023
2024 /** move unread buffer to start and clear rest for putting the rest into it */
2025 static void
2026 http_moveover_buffer(sldns_buffer* buf)
2027 {
2028         size_t pos = sldns_buffer_position(buf);
2029         size_t len = sldns_buffer_remaining(buf);
2030         sldns_buffer_clear(buf);
2031         memmove(sldns_buffer_begin(buf), sldns_buffer_at(buf, pos), len);
2032         sldns_buffer_set_position(buf, len);
2033 }
2034
2035 /** a http header is complete, process it */
2036 static int
2037 http_process_initial_header(struct comm_point* c)
2038 {
2039         char* line = http_header_line(c->buffer);
2040         if(!line) return 1;
2041         verbose(VERB_ALGO, "http header: %s", line);
2042         if(strncasecmp(line, "HTTP/1.1 ", 9) == 0) {
2043                 /* check returncode */
2044                 if(line[9] != '2') {
2045                         verbose(VERB_ALGO, "http bad status %s", line+9);
2046                         return 0;
2047                 }
2048         } else if(strncasecmp(line, "Content-Length: ", 16) == 0) {
2049                 if(!c->http_is_chunked)
2050                         c->tcp_byte_count = (size_t)atoi(line+16);
2051         } else if(strncasecmp(line, "Transfer-Encoding: chunked", 19+7) == 0) {
2052                 c->tcp_byte_count = 0;
2053                 c->http_is_chunked = 1;
2054         } else if(line[0] == 0) {
2055                 /* end of initial headers */
2056                 c->http_in_headers = 0;
2057                 if(c->http_is_chunked)
2058                         c->http_in_chunk_headers = 1;
2059                 /* remove header text from front of buffer
2060                  * the buffer is going to be used to return the data segment
2061                  * itself and we don't want the header to get returned
2062                  * prepended with it */
2063                 http_moveover_buffer(c->buffer);
2064                 sldns_buffer_flip(c->buffer);
2065                 return 1;
2066         }
2067         /* ignore other headers */
2068         return 1;
2069 }
2070
2071 /** a chunk header is complete, process it, return 0=fail, 1=continue next
2072  * header line, 2=done with chunked transfer*/
2073 static int
2074 http_process_chunk_header(struct comm_point* c)
2075 {
2076         char* line = http_header_line(c->buffer);
2077         if(!line) return 1;
2078         if(c->http_in_chunk_headers == 3) {
2079                 verbose(VERB_ALGO, "http chunk trailer: %s", line);
2080                 /* are we done ? */
2081                 if(line[0] == 0 && c->tcp_byte_count == 0) {
2082                         /* callback of http reader when NETEVENT_DONE,
2083                          * end of data, with no data in buffer */
2084                         sldns_buffer_set_position(c->buffer, 0);
2085                         sldns_buffer_set_limit(c->buffer, 0);
2086                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2087                         (void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2088                         /* return that we are done */
2089                         return 2;
2090                 }
2091                 if(line[0] == 0) {
2092                         /* continue with header of the next chunk */
2093                         c->http_in_chunk_headers = 1;
2094                         /* remove header text from front of buffer */
2095                         http_moveover_buffer(c->buffer);
2096                         sldns_buffer_flip(c->buffer);
2097                         return 1;
2098                 }
2099                 /* ignore further trail headers */
2100                 return 1;
2101         }
2102         verbose(VERB_ALGO, "http chunk header: %s", line);
2103         if(c->http_in_chunk_headers == 1) {
2104                 /* read chunked start line */
2105                 char* end = NULL;
2106                 c->tcp_byte_count = (size_t)strtol(line, &end, 16);
2107                 if(end == line)
2108                         return 0;
2109                 c->http_in_chunk_headers = 0;
2110                 /* remove header text from front of buffer */
2111                 http_moveover_buffer(c->buffer);
2112                 sldns_buffer_flip(c->buffer);
2113                 if(c->tcp_byte_count == 0) {
2114                         /* done with chunks, process chunk_trailer lines */
2115                         c->http_in_chunk_headers = 3;
2116                 }
2117                 return 1;
2118         }
2119         /* ignore other headers */
2120         return 1;
2121 }
2122
2123 /** handle nonchunked data segment */
2124 static int
2125 http_nonchunk_segment(struct comm_point* c)
2126 {
2127         /* c->buffer at position..limit has new data we read in.
2128          * the buffer itself is full of nonchunked data.
2129          * we are looking to read tcp_byte_count more data
2130          * and then the transfer is done. */
2131         size_t remainbufferlen;
2132         size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
2133         if(c->tcp_byte_count <= got_now) {
2134                 /* done, this is the last data fragment */
2135                 c->http_stored = 0;
2136                 sldns_buffer_set_position(c->buffer, 0);
2137                 fptr_ok(fptr_whitelist_comm_point(c->callback));
2138                 (void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2139                 return 1;
2140         }
2141         c->tcp_byte_count -= got_now;
2142         /* if we have the buffer space,
2143          * read more data collected into the buffer */
2144         remainbufferlen = sldns_buffer_capacity(c->buffer) -
2145                 sldns_buffer_limit(c->buffer);
2146         if(remainbufferlen >= c->tcp_byte_count ||
2147                 remainbufferlen >= 2048) {
2148                 size_t total = sldns_buffer_limit(c->buffer);
2149                 sldns_buffer_clear(c->buffer);
2150                 sldns_buffer_set_position(c->buffer, total);
2151                 c->http_stored = total;
2152                 /* return and wait to read more */
2153                 return 1;
2154         }
2155         /* call callback with this data amount, then
2156          * wait for more */
2157         c->http_stored = 0;
2158         sldns_buffer_set_position(c->buffer, 0);
2159         fptr_ok(fptr_whitelist_comm_point(c->callback));
2160         (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
2161         /* c->callback has to buffer_clear(c->buffer). */
2162         /* return and wait to read more */
2163         return 1;
2164 }
2165
2166 /** handle nonchunked data segment, return 0=fail, 1=wait, 2=process more */
2167 static int
2168 http_chunked_segment(struct comm_point* c)
2169 {
2170         /* the c->buffer has from position..limit new data we read. */
2171         /* the current chunk has length tcp_byte_count.
2172          * once we read that read more chunk headers.
2173          */
2174         size_t remainbufferlen;
2175         size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
2176         if(c->tcp_byte_count <= got_now) {
2177                 /* the chunk has completed (with perhaps some extra data
2178                  * from next chunk header and next chunk) */
2179                 /* save too much info into temp buffer */
2180                 size_t fraglen;
2181                 struct comm_reply repinfo;
2182                 c->http_stored = 0;
2183                 sldns_buffer_skip(c->buffer, (ssize_t)c->tcp_byte_count);
2184                 sldns_buffer_clear(c->http_temp);
2185                 sldns_buffer_write(c->http_temp,
2186                         sldns_buffer_current(c->buffer),
2187                         sldns_buffer_remaining(c->buffer));
2188                 sldns_buffer_flip(c->http_temp);
2189
2190                 /* callback with this fragment */
2191                 fraglen = sldns_buffer_position(c->buffer);
2192                 sldns_buffer_set_position(c->buffer, 0);
2193                 sldns_buffer_set_limit(c->buffer, fraglen);
2194                 repinfo = c->repinfo;
2195                 fptr_ok(fptr_whitelist_comm_point(c->callback));
2196                 (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &repinfo);
2197                 /* c->callback has to buffer_clear(). */
2198
2199                 /* is commpoint deleted? */
2200                 if(!repinfo.c) {
2201                         return 1;
2202                 }
2203                 /* copy waiting info */
2204                 sldns_buffer_clear(c->buffer);
2205                 sldns_buffer_write(c->buffer,
2206                         sldns_buffer_begin(c->http_temp),
2207                         sldns_buffer_remaining(c->http_temp));
2208                 sldns_buffer_flip(c->buffer);
2209                 /* process end of chunk trailer header lines, until
2210                  * an empty line */
2211                 c->http_in_chunk_headers = 3;
2212                 /* process more data in buffer (if any) */
2213                 return 2;
2214         }
2215         c->tcp_byte_count -= got_now;
2216
2217         /* if we have the buffer space,
2218          * read more data collected into the buffer */
2219         remainbufferlen = sldns_buffer_capacity(c->buffer) -
2220                 sldns_buffer_limit(c->buffer);
2221         if(remainbufferlen >= c->tcp_byte_count ||
2222                 remainbufferlen >= 2048) {
2223                 size_t total = sldns_buffer_limit(c->buffer);
2224                 sldns_buffer_clear(c->buffer);
2225                 sldns_buffer_set_position(c->buffer, total);
2226                 c->http_stored = total;
2227                 /* return and wait to read more */
2228                 return 1;
2229         }
2230
2231         /* callback of http reader for a new part of the data */
2232         c->http_stored = 0;
2233         sldns_buffer_set_position(c->buffer, 0);
2234         fptr_ok(fptr_whitelist_comm_point(c->callback));
2235         (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
2236         /* c->callback has to buffer_clear(c->buffer). */
2237         /* return and wait to read more */
2238         return 1;
2239 }
2240
2241 #ifdef HAVE_NGHTTP2
2242 /** Create new http2 session. Called when creating handling comm point. */
2243 struct http2_session* http2_session_create(struct comm_point* c)
2244 {
2245         struct http2_session* session = calloc(1, sizeof(*session));
2246         if(!session) {
2247                 log_err("malloc failure while creating http2 session");
2248                 return NULL;
2249         }
2250         session->c = c;
2251
2252         return session;
2253 }
2254 #endif
2255
2256 /** Delete http2 session. After closing connection or on error */
2257 void http2_session_delete(struct http2_session* h2_session)
2258 {
2259 #ifdef HAVE_NGHTTP2
2260         if(h2_session->callbacks)
2261                 nghttp2_session_callbacks_del(h2_session->callbacks);
2262         free(h2_session);
2263 #else
2264         (void)h2_session;
2265 #endif
2266 }
2267
2268 #ifdef HAVE_NGHTTP2
2269 struct http2_stream* http2_stream_create(int32_t stream_id)
2270 {
2271         struct http2_stream* h2_stream = calloc(1, sizeof(*h2_stream));
2272         if(!h2_stream) {
2273                 log_err("malloc failure while creating http2 stream");
2274                 return NULL;
2275         }
2276         h2_stream->stream_id = stream_id;
2277         return h2_stream;
2278 }
2279
2280 /** Delete http2 stream. After session delete or stream close callback */
2281 static void http2_stream_delete(struct http2_session* h2_session,
2282         struct http2_stream* h2_stream)
2283 {
2284         if(h2_stream->mesh_state) {
2285                 mesh_state_remove_reply(h2_stream->mesh, h2_stream->mesh_state,
2286                         h2_session->c);
2287                 h2_stream->mesh_state = NULL;
2288         }
2289         http2_req_stream_clear(h2_stream);
2290         free(h2_stream);
2291 }
2292 #endif
2293
2294 void http2_stream_add_meshstate(struct http2_stream* h2_stream,
2295         struct mesh_area* mesh, struct mesh_state* m)
2296 {
2297         h2_stream->mesh = mesh;
2298         h2_stream->mesh_state = m;
2299 }
2300
2301 /** delete http2 session server. After closing connection. */
2302 static void http2_session_server_delete(struct http2_session* h2_session)
2303 {
2304 #ifdef HAVE_NGHTTP2
2305         struct http2_stream* h2_stream, *next;
2306         nghttp2_session_del(h2_session->session); /* NULL input is fine */
2307         h2_session->session = NULL;
2308         for(h2_stream = h2_session->first_stream; h2_stream;) {
2309                 next = h2_stream->next;
2310                 http2_stream_delete(h2_session, h2_stream);
2311                 h2_stream = next;
2312         }
2313         h2_session->first_stream = NULL;
2314         h2_session->is_drop = 0;
2315         h2_session->postpone_drop = 0;
2316         h2_session->c->h2_stream = NULL;
2317 #endif
2318         (void)h2_session;
2319 }
2320
2321 #ifdef HAVE_NGHTTP2
2322 void http2_session_add_stream(struct http2_session* h2_session,
2323         struct http2_stream* h2_stream)
2324 {
2325         if(h2_session->first_stream)
2326                 h2_session->first_stream->prev = h2_stream;
2327         h2_stream->next = h2_session->first_stream;
2328         h2_session->first_stream = h2_stream;
2329 }
2330
2331 /** remove stream from session linked list. After stream close callback or
2332  * closing connection */
2333 void http2_session_remove_stream(struct http2_session* h2_session,
2334         struct http2_stream* h2_stream)
2335 {
2336         if(h2_stream->prev)
2337                 h2_stream->prev->next = h2_stream->next;
2338         else
2339                 h2_session->first_stream = h2_stream->next;
2340         if(h2_stream->next)
2341                 h2_stream->next->prev = h2_stream->prev;
2342
2343 }
2344
2345 int http2_stream_close_cb(nghttp2_session* ATTR_UNUSED(session),
2346         int32_t stream_id, uint32_t ATTR_UNUSED(error_code), void* cb_arg)
2347 {
2348         struct http2_stream* h2_stream;
2349         struct http2_session* h2_session = (struct http2_session*)cb_arg;
2350         if(!(h2_stream = nghttp2_session_get_stream_user_data(
2351                 h2_session->session, stream_id))) {
2352                 return 0;
2353         }
2354         http2_session_remove_stream(h2_session, h2_stream);
2355         http2_stream_delete(h2_session, h2_stream);
2356         return 0;
2357 }
2358
2359 ssize_t http2_recv_cb(nghttp2_session* ATTR_UNUSED(session), uint8_t* buf,
2360         size_t len, int ATTR_UNUSED(flags), void* cb_arg)
2361 {
2362 #ifdef HAVE_SSL
2363         struct http2_session* h2_session = (struct http2_session*)cb_arg;
2364         int r;
2365
2366         log_assert(h2_session->c->type == comm_http);
2367         log_assert(h2_session->c->h2_session);
2368
2369         if(!h2_session->c->ssl)
2370                 return 0;
2371
2372         ERR_clear_error();
2373         r = SSL_read(h2_session->c->ssl, buf, len);
2374         if(r <= 0) {
2375                 int want = SSL_get_error(h2_session->c->ssl, r);
2376                 if(want == SSL_ERROR_ZERO_RETURN) {
2377                         return NGHTTP2_ERR_EOF;
2378                 } else if(want == SSL_ERROR_WANT_READ) {
2379                         return NGHTTP2_ERR_WOULDBLOCK;
2380                 } else if(want == SSL_ERROR_WANT_WRITE) {
2381                         h2_session->c->ssl_shake_state = comm_ssl_shake_hs_write;
2382                         comm_point_listen_for_rw(h2_session->c, 0, 1);
2383                         return NGHTTP2_ERR_WOULDBLOCK;
2384                 } else if(want == SSL_ERROR_SYSCALL) {
2385 #ifdef ECONNRESET
2386                         if(errno == ECONNRESET && verbosity < 2)
2387                                 return NGHTTP2_ERR_CALLBACK_FAILURE;
2388 #endif
2389                         if(errno != 0)
2390                                 log_err("SSL_read syscall: %s",
2391                                         strerror(errno));
2392                         return NGHTTP2_ERR_CALLBACK_FAILURE;
2393                 }
2394                 log_crypto_err("could not SSL_read");
2395                 return NGHTTP2_ERR_CALLBACK_FAILURE;
2396         }
2397         return r;
2398 #else
2399         (void)buf;
2400         (void)len;
2401         (void)cb_arg;
2402         return -1;
2403 #endif
2404 }
2405 #endif /* HAVE_NGHTTP2 */
2406
2407 /** Handle http2 read */
2408 static int
2409 comm_point_http2_handle_read(int ATTR_UNUSED(fd), struct comm_point* c)
2410 {
2411 #ifdef HAVE_NGHTTP2
2412         int ret;
2413         log_assert(c->h2_session);
2414         log_assert(c->ssl);
2415
2416         /* reading until recv cb returns NGHTTP2_ERR_WOULDBLOCK */
2417         ret = nghttp2_session_recv(c->h2_session->session);
2418         if(ret) {
2419                 if(ret != NGHTTP2_ERR_EOF &&
2420                         ret != NGHTTP2_ERR_CALLBACK_FAILURE) {
2421                         verbose(VERB_QUERY, "http2: session_recv failed, "
2422                                 "error: %s", nghttp2_strerror(ret));
2423                 }
2424                 return 0;
2425         }
2426         if(nghttp2_session_want_write(c->h2_session->session)) {
2427                 c->tcp_is_reading = 0;
2428                 comm_point_stop_listening(c);
2429                 comm_point_start_listening(c, -1, c->tcp_timeout_msec);
2430         } else if(!nghttp2_session_want_read(c->h2_session->session))
2431                 return 0; /* connection can be closed */
2432         return 1;
2433 #else
2434         (void)c;
2435         return 0;
2436 #endif
2437 }
2438
2439 /**
2440  * Handle http reading callback.
2441  * @param fd: file descriptor of socket.
2442  * @param c: comm point to read from into buffer.
2443  * @return: 0 on error
2444  */
2445 static int
2446 comm_point_http_handle_read(int fd, struct comm_point* c)
2447 {
2448         log_assert(c->type == comm_http);
2449         log_assert(fd != -1);
2450
2451         /* if we are in ssl handshake, handle SSL handshake */
2452 #ifdef HAVE_SSL
2453         if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
2454                 if(!ssl_handshake(c))
2455                         return 0;
2456                 if(c->ssl_shake_state != comm_ssl_shake_none)
2457                         return 1;
2458         }
2459 #endif /* HAVE_SSL */
2460
2461         if(!c->tcp_is_reading)
2462                 return 1;
2463
2464         if(c->use_h2) {
2465                 return comm_point_http2_handle_read(fd, c);
2466         }
2467
2468         /* http version is <= http/1.1 */
2469
2470         if(c->http_min_version >= http_version_2) {
2471                 /* HTTP/2 failed, not allowed to use lower version. */
2472                 return 0;
2473         }
2474
2475         /* read more data */
2476         if(c->ssl) {
2477                 if(!ssl_http_read_more(c))
2478                         return 0;
2479         } else {
2480                 if(!http_read_more(fd, c))
2481                         return 0;
2482         }
2483
2484         sldns_buffer_flip(c->buffer);
2485
2486         while(sldns_buffer_remaining(c->buffer) > 0) {
2487                 /* Handle HTTP/1.x data */
2488                 /* if we are reading headers, read more headers */
2489                 if(c->http_in_headers || c->http_in_chunk_headers) {
2490                         /* if header is done, process the header */
2491                         if(!http_header_done(c->buffer)) {
2492                                 /* copy remaining data to front of buffer
2493                                  * and set rest for writing into it */
2494                                 http_moveover_buffer(c->buffer);
2495                                 /* return and wait to read more */
2496                                 return 1;
2497                         }
2498                         if(!c->http_in_chunk_headers) {
2499                                 /* process initial headers */
2500                                 if(!http_process_initial_header(c))
2501                                         return 0;
2502                         } else {
2503                                 /* process chunk headers */
2504                                 int r = http_process_chunk_header(c);
2505                                 if(r == 0) return 0;
2506                                 if(r == 2) return 1; /* done */
2507                                 /* r == 1, continue */
2508                         }
2509                         /* see if we have more to process */
2510                         continue;
2511                 }
2512
2513                 if(!c->http_is_chunked) {
2514                         /* if we are reading nonchunks, process that*/
2515                         return http_nonchunk_segment(c);
2516                 } else {
2517                         /* if we are reading chunks, read the chunk */
2518                         int r = http_chunked_segment(c);
2519                         if(r == 0) return 0;
2520                         if(r == 1) return 1;
2521                         continue;
2522                 }
2523         }
2524         /* broke out of the loop; could not process header instead need
2525          * to read more */
2526         /* moveover any remaining data and read more data */
2527         http_moveover_buffer(c->buffer);
2528         /* return and wait to read more */
2529         return 1;
2530 }
2531
2532 /** check pending connect for http */
2533 static int
2534 http_check_connect(int fd, struct comm_point* c)
2535 {
2536         /* check for pending error from nonblocking connect */
2537         /* from Stevens, unix network programming, vol1, 3rd ed, p450*/
2538         int error = 0;
2539         socklen_t len = (socklen_t)sizeof(error);
2540         if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
2541                 &len) < 0){
2542 #ifndef USE_WINSOCK
2543                 error = errno; /* on solaris errno is error */
2544 #else /* USE_WINSOCK */
2545                 error = WSAGetLastError();
2546 #endif
2547         }
2548 #ifndef USE_WINSOCK
2549 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
2550         if(error == EINPROGRESS || error == EWOULDBLOCK)
2551                 return 1; /* try again later */
2552         else
2553 #endif
2554         if(error != 0 && verbosity < 2)
2555                 return 0; /* silence lots of chatter in the logs */
2556         else if(error != 0) {
2557                 log_err_addr("http connect", strerror(error),
2558                         &c->repinfo.addr, c->repinfo.addrlen);
2559 #else /* USE_WINSOCK */
2560         /* examine error */
2561         if(error == WSAEINPROGRESS)
2562                 return 1;
2563         else if(error == WSAEWOULDBLOCK) {
2564                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2565                 return 1;
2566         } else if(error != 0 && verbosity < 2)
2567                 return 0;
2568         else if(error != 0) {
2569                 log_err_addr("http connect", wsa_strerror(error),
2570                         &c->repinfo.addr, c->repinfo.addrlen);
2571 #endif /* USE_WINSOCK */
2572                 return 0;
2573         }
2574         /* keep on processing this socket */
2575         return 2;
2576 }
2577
2578 /** write more data for http (with ssl) */
2579 static int
2580 ssl_http_write_more(struct comm_point* c)
2581 {
2582 #ifdef HAVE_SSL
2583         int r;
2584         log_assert(sldns_buffer_remaining(c->buffer) > 0);
2585         ERR_clear_error();
2586         r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
2587                 (int)sldns_buffer_remaining(c->buffer));
2588         if(r <= 0) {
2589                 int want = SSL_get_error(c->ssl, r);
2590                 if(want == SSL_ERROR_ZERO_RETURN) {
2591                         return 0; /* closed */
2592                 } else if(want == SSL_ERROR_WANT_READ) {
2593                         c->ssl_shake_state = comm_ssl_shake_hs_read;
2594                         comm_point_listen_for_rw(c, 1, 0);
2595                         return 1; /* wait for read condition */
2596                 } else if(want == SSL_ERROR_WANT_WRITE) {
2597                         return 1; /* write more later */
2598                 } else if(want == SSL_ERROR_SYSCALL) {
2599 #ifdef EPIPE
2600                         if(errno == EPIPE && verbosity < 2)
2601                                 return 0; /* silence 'broken pipe' */
2602 #endif
2603                         if(errno != 0)
2604                                 log_err("SSL_write syscall: %s",
2605                                         strerror(errno));
2606                         return 0;
2607                 }
2608                 log_crypto_err("could not SSL_write");
2609                 return 0;
2610         }
2611         sldns_buffer_skip(c->buffer, (ssize_t)r);
2612         return 1;
2613 #else
2614         (void)c;
2615         return 0;
2616 #endif /* HAVE_SSL */
2617 }
2618
2619 /** write more data for http */
2620 static int
2621 http_write_more(int fd, struct comm_point* c)
2622 {
2623         ssize_t r;
2624         log_assert(sldns_buffer_remaining(c->buffer) > 0);
2625         r = send(fd, (void*)sldns_buffer_current(c->buffer),
2626                 sldns_buffer_remaining(c->buffer), 0);
2627         if(r == -1) {
2628 #ifndef USE_WINSOCK
2629                 if(errno == EINTR || errno == EAGAIN)
2630                         return 1;
2631 #else
2632                 if(WSAGetLastError() == WSAEINPROGRESS)
2633                         return 1;
2634                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
2635                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2636                         return 1;
2637                 }
2638 #endif
2639                 log_err_addr("http send r", sock_strerror(errno),
2640                         &c->repinfo.addr, c->repinfo.addrlen);
2641                 return 0;
2642         }
2643         sldns_buffer_skip(c->buffer, r);
2644         return 1;
2645 }
2646
2647 #ifdef HAVE_NGHTTP2
2648 ssize_t http2_send_cb(nghttp2_session* ATTR_UNUSED(session), const uint8_t* buf,
2649         size_t len, int ATTR_UNUSED(flags), void* cb_arg)
2650 {
2651 #ifdef HAVE_SSL
2652         int r;
2653         struct http2_session* h2_session = (struct http2_session*)cb_arg;
2654         log_assert(h2_session->c->type == comm_http);
2655         log_assert(h2_session->c->h2_session);
2656
2657         if(!h2_session->c->ssl)
2658                 return 0;
2659
2660         ERR_clear_error();
2661         r = SSL_write(h2_session->c->ssl, buf, len);
2662         if(r <= 0) {
2663                 int want = SSL_get_error(h2_session->c->ssl, r);
2664                 if(want == SSL_ERROR_ZERO_RETURN) {
2665                         return NGHTTP2_ERR_CALLBACK_FAILURE;
2666                 } else if(want == SSL_ERROR_WANT_READ) {
2667                         h2_session->c->ssl_shake_state = comm_ssl_shake_hs_read;
2668                         comm_point_listen_for_rw(h2_session->c, 1, 0);
2669                         return NGHTTP2_ERR_WOULDBLOCK;
2670                 } else if(want == SSL_ERROR_WANT_WRITE) {
2671                         return NGHTTP2_ERR_WOULDBLOCK;
2672                 } else if(want == SSL_ERROR_SYSCALL) {
2673 #ifdef EPIPE
2674                         if(errno == EPIPE && verbosity < 2)
2675                                 return NGHTTP2_ERR_CALLBACK_FAILURE;
2676 #endif
2677                         if(errno != 0)
2678                                 log_err("SSL_write syscall: %s",
2679                                         strerror(errno));
2680                         return NGHTTP2_ERR_CALLBACK_FAILURE;
2681                 }
2682                 log_crypto_err("could not SSL_write");
2683                 return NGHTTP2_ERR_CALLBACK_FAILURE;
2684         }
2685         return r;
2686 #else
2687         (void)buf;
2688         (void)len;
2689         (void)cb_arg;
2690         return -1;
2691 #endif
2692 }
2693 #endif /* HAVE_NGHTTP2 */
2694
2695 /** Handle http2 writing */
2696 static int
2697 comm_point_http2_handle_write(int ATTR_UNUSED(fd), struct comm_point* c)
2698 {
2699 #ifdef HAVE_NGHTTP2
2700         int ret;
2701         log_assert(c->h2_session);
2702         log_assert(c->ssl);
2703
2704         ret = nghttp2_session_send(c->h2_session->session);
2705         if(ret) {
2706                 verbose(VERB_QUERY, "http2: session_send failed, "
2707                         "error: %s", nghttp2_strerror(ret));
2708                 return 0;
2709         }
2710
2711         if(nghttp2_session_want_read(c->h2_session->session)) {
2712                 c->tcp_is_reading = 1;
2713                 comm_point_stop_listening(c);
2714                 comm_point_start_listening(c, -1, c->tcp_timeout_msec);
2715         } else if(!nghttp2_session_want_write(c->h2_session->session))
2716                 return 0; /* connection can be closed */
2717         return 1;
2718 #else
2719         (void)c;
2720         return 0;
2721 #endif
2722 }
2723
2724 /**
2725  * Handle http writing callback.
2726  * @param fd: file descriptor of socket.
2727  * @param c: comm point to write buffer out of.
2728  * @return: 0 on error
2729  */
2730 static int
2731 comm_point_http_handle_write(int fd, struct comm_point* c)
2732 {
2733         log_assert(c->type == comm_http);
2734         log_assert(fd != -1);
2735
2736         /* check pending connect errors, if that fails, we wait for more,
2737          * or we can continue to write contents */
2738         if(c->tcp_check_nb_connect) {
2739                 int r = http_check_connect(fd, c);
2740                 if(r == 0) return 0;
2741                 if(r == 1) return 1;
2742                 c->tcp_check_nb_connect = 0;
2743         }
2744         /* if we are in ssl handshake, handle SSL handshake */
2745 #ifdef HAVE_SSL
2746         if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
2747                 if(!ssl_handshake(c))
2748                         return 0;
2749                 if(c->ssl_shake_state != comm_ssl_shake_none)
2750                         return 1;
2751         }
2752 #endif /* HAVE_SSL */
2753         if(c->tcp_is_reading)
2754                 return 1;
2755
2756         if(c->use_h2) {
2757                 return comm_point_http2_handle_write(fd, c);
2758         }
2759
2760         /* http version is <= http/1.1 */
2761
2762         if(c->http_min_version >= http_version_2) {
2763                 /* HTTP/2 failed, not allowed to use lower version. */
2764                 return 0;
2765         }
2766
2767         /* if we are writing, write more */
2768         if(c->ssl) {
2769                 if(!ssl_http_write_more(c))
2770                         return 0;
2771         } else {
2772                 if(!http_write_more(fd, c))
2773                         return 0;
2774         }
2775
2776         /* we write a single buffer contents, that can contain
2777          * the http request, and then flip to read the results */
2778         /* see if write is done */
2779         if(sldns_buffer_remaining(c->buffer) == 0) {
2780                 sldns_buffer_clear(c->buffer);
2781                 if(c->tcp_do_toggle_rw)
2782                         c->tcp_is_reading = 1;
2783                 c->tcp_byte_count = 0;
2784                 /* switch from listening(write) to listening(read) */
2785                 comm_point_stop_listening(c);
2786                 comm_point_start_listening(c, -1, -1);
2787         }
2788         return 1;
2789 }
2790
2791 void
2792 comm_point_http_handle_callback(int fd, short event, void* arg)
2793 {
2794         struct comm_point* c = (struct comm_point*)arg;
2795         log_assert(c->type == comm_http);
2796         ub_comm_base_now(c->ev->base);
2797
2798         if(event&UB_EV_TIMEOUT) {
2799                 verbose(VERB_QUERY, "http took too long, dropped");
2800                 reclaim_http_handler(c);
2801                 if(!c->tcp_do_close) {
2802                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2803                         (void)(*c->callback)(c, c->cb_arg,
2804                                 NETEVENT_TIMEOUT, NULL);
2805                 }
2806                 return;
2807         }
2808         if(event&UB_EV_READ) {
2809                 if(!comm_point_http_handle_read(fd, c)) {
2810                         reclaim_http_handler(c);
2811                         if(!c->tcp_do_close) {
2812                                 fptr_ok(fptr_whitelist_comm_point(
2813                                         c->callback));
2814                                 (void)(*c->callback)(c, c->cb_arg,
2815                                         NETEVENT_CLOSED, NULL);
2816                         }
2817                 }
2818                 return;
2819         }
2820         if(event&UB_EV_WRITE) {
2821                 if(!comm_point_http_handle_write(fd, c)) {
2822                         reclaim_http_handler(c);
2823                         if(!c->tcp_do_close) {
2824                                 fptr_ok(fptr_whitelist_comm_point(
2825                                         c->callback));
2826                                 (void)(*c->callback)(c, c->cb_arg,
2827                                         NETEVENT_CLOSED, NULL);
2828                         }
2829                 }
2830                 return;
2831         }
2832         log_err("Ignored event %d for httphdl.", event);
2833 }
2834
2835 void comm_point_local_handle_callback(int fd, short event, void* arg)
2836 {
2837         struct comm_point* c = (struct comm_point*)arg;
2838         log_assert(c->type == comm_local);
2839         ub_comm_base_now(c->ev->base);
2840
2841         if(event&UB_EV_READ) {
2842                 if(!comm_point_tcp_handle_read(fd, c, 1)) {
2843                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2844                         (void)(*c->callback)(c, c->cb_arg, NETEVENT_CLOSED,
2845                                 NULL);
2846                 }
2847                 return;
2848         }
2849         log_err("Ignored event %d for localhdl.", event);
2850 }
2851
2852 void comm_point_raw_handle_callback(int ATTR_UNUSED(fd),
2853         short event, void* arg)
2854 {
2855         struct comm_point* c = (struct comm_point*)arg;
2856         int err = NETEVENT_NOERROR;
2857         log_assert(c->type == comm_raw);
2858         ub_comm_base_now(c->ev->base);
2859
2860         if(event&UB_EV_TIMEOUT)
2861                 err = NETEVENT_TIMEOUT;
2862         fptr_ok(fptr_whitelist_comm_point_raw(c->callback));
2863         (void)(*c->callback)(c, c->cb_arg, err, NULL);
2864 }
2865
2866 struct comm_point*
2867 comm_point_create_udp(struct comm_base *base, int fd, sldns_buffer* buffer,
2868         comm_point_callback_type* callback, void* callback_arg)
2869 {
2870         struct comm_point* c = (struct comm_point*)calloc(1,
2871                 sizeof(struct comm_point));
2872         short evbits;
2873         if(!c)
2874                 return NULL;
2875         c->ev = (struct internal_event*)calloc(1,
2876                 sizeof(struct internal_event));
2877         if(!c->ev) {
2878                 free(c);
2879                 return NULL;
2880         }
2881         c->ev->base = base;
2882         c->fd = fd;
2883         c->buffer = buffer;
2884         c->timeout = NULL;
2885         c->tcp_is_reading = 0;
2886         c->tcp_byte_count = 0;
2887         c->tcp_parent = NULL;
2888         c->max_tcp_count = 0;
2889         c->cur_tcp_count = 0;
2890         c->tcp_handlers = NULL;
2891         c->tcp_free = NULL;
2892         c->type = comm_udp;
2893         c->tcp_do_close = 0;
2894         c->do_not_close = 0;
2895         c->tcp_do_toggle_rw = 0;
2896         c->tcp_check_nb_connect = 0;
2897 #ifdef USE_MSG_FASTOPEN
2898         c->tcp_do_fastopen = 0;
2899 #endif
2900 #ifdef USE_DNSCRYPT
2901         c->dnscrypt = 0;
2902         c->dnscrypt_buffer = buffer;
2903 #endif
2904         c->inuse = 0;
2905         c->callback = callback;
2906         c->cb_arg = callback_arg;
2907         evbits = UB_EV_READ | UB_EV_PERSIST;
2908         /* ub_event stuff */
2909         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2910                 comm_point_udp_callback, c);
2911         if(c->ev->ev == NULL) {
2912                 log_err("could not baseset udp event");
2913                 comm_point_delete(c);
2914                 return NULL;
2915         }
2916         if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
2917                 log_err("could not add udp event");
2918                 comm_point_delete(c);
2919                 return NULL;
2920         }
2921         return c;
2922 }
2923
2924 struct comm_point*
2925 comm_point_create_udp_ancil(struct comm_base *base, int fd,
2926         sldns_buffer* buffer,
2927         comm_point_callback_type* callback, void* callback_arg)
2928 {
2929         struct comm_point* c = (struct comm_point*)calloc(1,
2930                 sizeof(struct comm_point));
2931         short evbits;
2932         if(!c)
2933                 return NULL;
2934         c->ev = (struct internal_event*)calloc(1,
2935                 sizeof(struct internal_event));
2936         if(!c->ev) {
2937                 free(c);
2938                 return NULL;
2939         }
2940         c->ev->base = base;
2941         c->fd = fd;
2942         c->buffer = buffer;
2943         c->timeout = NULL;
2944         c->tcp_is_reading = 0;
2945         c->tcp_byte_count = 0;
2946         c->tcp_parent = NULL;
2947         c->max_tcp_count = 0;
2948         c->cur_tcp_count = 0;
2949         c->tcp_handlers = NULL;
2950         c->tcp_free = NULL;
2951         c->type = comm_udp;
2952         c->tcp_do_close = 0;
2953         c->do_not_close = 0;
2954 #ifdef USE_DNSCRYPT
2955         c->dnscrypt = 0;
2956         c->dnscrypt_buffer = buffer;
2957 #endif
2958         c->inuse = 0;
2959         c->tcp_do_toggle_rw = 0;
2960         c->tcp_check_nb_connect = 0;
2961 #ifdef USE_MSG_FASTOPEN
2962         c->tcp_do_fastopen = 0;
2963 #endif
2964         c->callback = callback;
2965         c->cb_arg = callback_arg;
2966         evbits = UB_EV_READ | UB_EV_PERSIST;
2967         /* ub_event stuff */
2968         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
2969                 comm_point_udp_ancil_callback, c);
2970         if(c->ev->ev == NULL) {
2971                 log_err("could not baseset udp event");
2972                 comm_point_delete(c);
2973                 return NULL;
2974         }
2975         if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
2976                 log_err("could not add udp event");
2977                 comm_point_delete(c);
2978                 return NULL;
2979         }
2980         return c;
2981 }
2982
2983 static struct comm_point*
2984 comm_point_create_tcp_handler(struct comm_base *base,
2985         struct comm_point* parent, size_t bufsize,
2986         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
2987         void* callback_arg)
2988 {
2989         struct comm_point* c = (struct comm_point*)calloc(1,
2990                 sizeof(struct comm_point));
2991         short evbits;
2992         if(!c)
2993                 return NULL;
2994         c->ev = (struct internal_event*)calloc(1,
2995                 sizeof(struct internal_event));
2996         if(!c->ev) {
2997                 free(c);
2998                 return NULL;
2999         }
3000         c->ev->base = base;
3001         c->fd = -1;
3002         c->buffer = sldns_buffer_new(bufsize);
3003         if(!c->buffer) {
3004                 free(c->ev);
3005                 free(c);
3006                 return NULL;
3007         }
3008         c->timeout = (struct timeval*)malloc(sizeof(struct timeval));
3009         if(!c->timeout) {
3010                 sldns_buffer_free(c->buffer);
3011                 free(c->ev);
3012                 free(c);
3013                 return NULL;
3014         }
3015         c->tcp_is_reading = 0;
3016         c->tcp_byte_count = 0;
3017         c->tcp_parent = parent;
3018         c->tcp_timeout_msec = parent->tcp_timeout_msec;
3019         c->tcp_conn_limit = parent->tcp_conn_limit;
3020         c->tcl_addr = NULL;
3021         c->tcp_keepalive = 0;
3022         c->max_tcp_count = 0;
3023         c->cur_tcp_count = 0;
3024         c->tcp_handlers = NULL;
3025         c->tcp_free = NULL;
3026         c->type = comm_tcp;
3027         c->tcp_do_close = 0;
3028         c->do_not_close = 0;
3029         c->tcp_do_toggle_rw = 1;
3030         c->tcp_check_nb_connect = 0;
3031 #ifdef USE_MSG_FASTOPEN
3032         c->tcp_do_fastopen = 0;
3033 #endif
3034 #ifdef USE_DNSCRYPT
3035         c->dnscrypt = 0;
3036         /* We don't know just yet if this is a dnscrypt channel. Allocation
3037          * will be done when handling the callback. */
3038         c->dnscrypt_buffer = c->buffer;
3039 #endif
3040         c->repinfo.c = c;
3041         c->callback = callback;
3042         c->cb_arg = callback_arg;
3043         if(spoolbuf) {
3044                 c->tcp_req_info = tcp_req_info_create(spoolbuf);
3045                 if(!c->tcp_req_info) {
3046                         log_err("could not create tcp commpoint");
3047                         sldns_buffer_free(c->buffer);
3048                         free(c->timeout);
3049                         free(c->ev);
3050                         free(c);
3051                         return NULL;
3052                 }
3053                 c->tcp_req_info->cp = c;
3054                 c->tcp_do_close = 1;
3055                 c->tcp_do_toggle_rw = 0;
3056         }
3057         /* add to parent free list */
3058         c->tcp_free = parent->tcp_free;
3059         parent->tcp_free = c;
3060         /* ub_event stuff */
3061         evbits = UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT;
3062         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3063                 comm_point_tcp_handle_callback, c);
3064         if(c->ev->ev == NULL)
3065         {
3066                 log_err("could not basetset tcphdl event");
3067                 parent->tcp_free = c->tcp_free;
3068                 tcp_req_info_delete(c->tcp_req_info);
3069                 sldns_buffer_free(c->buffer);
3070                 free(c->timeout);
3071                 free(c->ev);
3072                 free(c);
3073                 return NULL;
3074         }
3075         return c;
3076 }
3077
3078 static struct comm_point*
3079 comm_point_create_http_handler(struct comm_base *base,
3080         struct comm_point* parent, size_t bufsize, int harden_large_queries,
3081         uint32_t http_max_streams, char* http_endpoint,
3082         comm_point_callback_type* callback, void* callback_arg)
3083 {
3084         struct comm_point* c = (struct comm_point*)calloc(1,
3085                 sizeof(struct comm_point));
3086         short evbits;
3087         if(!c)
3088                 return NULL;
3089         c->ev = (struct internal_event*)calloc(1,
3090                 sizeof(struct internal_event));
3091         if(!c->ev) {
3092                 free(c);
3093                 return NULL;
3094         }
3095         c->ev->base = base;
3096         c->fd = -1;
3097         c->buffer = sldns_buffer_new(bufsize);
3098         if(!c->buffer) {
3099                 free(c->ev);
3100                 free(c);
3101                 return NULL;
3102         }
3103         c->timeout = (struct timeval*)malloc(sizeof(struct timeval));
3104         if(!c->timeout) {
3105                 sldns_buffer_free(c->buffer);
3106                 free(c->ev);
3107                 free(c);
3108                 return NULL;
3109         }
3110         c->tcp_is_reading = 0;
3111         c->tcp_byte_count = 0;
3112         c->tcp_parent = parent;
3113         c->tcp_timeout_msec = parent->tcp_timeout_msec;
3114         c->tcp_conn_limit = parent->tcp_conn_limit;
3115         c->tcl_addr = NULL;
3116         c->tcp_keepalive = 0;
3117         c->max_tcp_count = 0;
3118         c->cur_tcp_count = 0;
3119         c->tcp_handlers = NULL;
3120         c->tcp_free = NULL;
3121         c->type = comm_http;
3122         c->tcp_do_close = 1;
3123         c->do_not_close = 0;
3124         c->tcp_do_toggle_rw = 1; /* will be set to 0 after http2 upgrade */
3125         c->tcp_check_nb_connect = 0;
3126 #ifdef USE_MSG_FASTOPEN
3127         c->tcp_do_fastopen = 0;
3128 #endif
3129 #ifdef USE_DNSCRYPT
3130         c->dnscrypt = 0;
3131         c->dnscrypt_buffer = NULL;
3132 #endif
3133         c->repinfo.c = c;
3134         c->callback = callback;
3135         c->cb_arg = callback_arg;
3136
3137         c->http_min_version = http_version_2;
3138         c->http2_stream_max_qbuffer_size = bufsize;
3139         if(harden_large_queries && bufsize > 512)
3140                 c->http2_stream_max_qbuffer_size = 512;
3141         c->http2_max_streams = http_max_streams;
3142         if(!(c->http_endpoint = strdup(http_endpoint))) {
3143                 log_err("could not strdup http_endpoint");
3144                 sldns_buffer_free(c->buffer);
3145                 free(c->timeout);
3146                 free(c->ev);
3147                 free(c);
3148                 return NULL;
3149         }
3150         c->use_h2 = 0;
3151 #ifdef HAVE_NGHTTP2
3152         if(!(c->h2_session = http2_session_create(c))) {
3153                 log_err("could not create http2 session");
3154                 free(c->http_endpoint);
3155                 sldns_buffer_free(c->buffer);
3156                 free(c->timeout);
3157                 free(c->ev);
3158                 free(c);
3159                 return NULL;
3160         }
3161         if(!(c->h2_session->callbacks = http2_req_callbacks_create())) {
3162                 log_err("could not create http2 callbacks");
3163                 http2_session_delete(c->h2_session);
3164                 free(c->http_endpoint);
3165                 sldns_buffer_free(c->buffer);
3166                 free(c->timeout);
3167                 free(c->ev);
3168                 free(c);
3169                 return NULL;
3170         }
3171 #endif
3172
3173         /* add to parent free list */
3174         c->tcp_free = parent->tcp_free;
3175         parent->tcp_free = c;
3176         /* ub_event stuff */
3177         evbits = UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT;
3178         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3179                 comm_point_http_handle_callback, c);
3180         if(c->ev->ev == NULL)
3181         {
3182                 log_err("could not set http handler event");
3183                 parent->tcp_free = c->tcp_free;
3184                 http2_session_delete(c->h2_session);
3185                 sldns_buffer_free(c->buffer);
3186                 free(c->timeout);
3187                 free(c->ev);
3188                 free(c);
3189                 return NULL;
3190         }
3191         return c;
3192 }
3193
3194 struct comm_point*
3195 comm_point_create_tcp(struct comm_base *base, int fd, int num,
3196         int idle_timeout, int harden_large_queries,
3197         uint32_t http_max_streams, char* http_endpoint,
3198         struct tcl_list* tcp_conn_limit, size_t bufsize,
3199         struct sldns_buffer* spoolbuf, enum listen_type port_type,
3200         comm_point_callback_type* callback, void* callback_arg)
3201 {
3202         struct comm_point* c = (struct comm_point*)calloc(1,
3203                 sizeof(struct comm_point));
3204         short evbits;
3205         int i;
3206         /* first allocate the TCP accept listener */
3207         if(!c)
3208                 return NULL;
3209         c->ev = (struct internal_event*)calloc(1,
3210                 sizeof(struct internal_event));
3211         if(!c->ev) {
3212                 free(c);
3213                 return NULL;
3214         }
3215         c->ev->base = base;
3216         c->fd = fd;
3217         c->buffer = NULL;
3218         c->timeout = NULL;
3219         c->tcp_is_reading = 0;
3220         c->tcp_byte_count = 0;
3221         c->tcp_timeout_msec = idle_timeout;
3222         c->tcp_conn_limit = tcp_conn_limit;
3223         c->tcl_addr = NULL;
3224         c->tcp_keepalive = 0;
3225         c->tcp_parent = NULL;
3226         c->max_tcp_count = num;
3227         c->cur_tcp_count = 0;
3228         c->tcp_handlers = (struct comm_point**)calloc((size_t)num,
3229                 sizeof(struct comm_point*));
3230         if(!c->tcp_handlers) {
3231                 free(c->ev);
3232                 free(c);
3233                 return NULL;
3234         }
3235         c->tcp_free = NULL;
3236         c->type = comm_tcp_accept;
3237         c->tcp_do_close = 0;
3238         c->do_not_close = 0;
3239         c->tcp_do_toggle_rw = 0;
3240         c->tcp_check_nb_connect = 0;
3241 #ifdef USE_MSG_FASTOPEN
3242         c->tcp_do_fastopen = 0;
3243 #endif
3244 #ifdef USE_DNSCRYPT
3245         c->dnscrypt = 0;
3246         c->dnscrypt_buffer = NULL;
3247 #endif
3248         c->callback = NULL;
3249         c->cb_arg = NULL;
3250         evbits = UB_EV_READ | UB_EV_PERSIST;
3251         /* ub_event stuff */
3252         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3253                 comm_point_tcp_accept_callback, c);
3254         if(c->ev->ev == NULL) {
3255                 log_err("could not baseset tcpacc event");
3256                 comm_point_delete(c);
3257                 return NULL;
3258         }
3259         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3260                 log_err("could not add tcpacc event");
3261                 comm_point_delete(c);
3262                 return NULL;
3263         }
3264         /* now prealloc the handlers */
3265         for(i=0; i<num; i++) {
3266                 if(port_type == listen_type_tcp ||
3267                         port_type == listen_type_ssl ||
3268                         port_type == listen_type_tcp_dnscrypt) {
3269                         c->tcp_handlers[i] = comm_point_create_tcp_handler(base,
3270                                 c, bufsize, spoolbuf, callback, callback_arg);
3271                 } else if(port_type == listen_type_http) {
3272                         c->tcp_handlers[i] = comm_point_create_http_handler(
3273                                 base, c, bufsize, harden_large_queries,
3274                                 http_max_streams, http_endpoint,
3275                                 callback, callback_arg);
3276                 }
3277                 else {
3278                         log_err("could not create tcp handler, unknown listen "
3279                                 "type");
3280                         return NULL;
3281                 }
3282                 if(!c->tcp_handlers[i]) {
3283                         comm_point_delete(c);
3284                         return NULL;
3285                 }
3286         }
3287
3288         return c;
3289 }
3290
3291 struct comm_point*
3292 comm_point_create_tcp_out(struct comm_base *base, size_t bufsize,
3293         comm_point_callback_type* callback, void* callback_arg)
3294 {
3295         struct comm_point* c = (struct comm_point*)calloc(1,
3296                 sizeof(struct comm_point));
3297         short evbits;
3298         if(!c)
3299                 return NULL;
3300         c->ev = (struct internal_event*)calloc(1,
3301                 sizeof(struct internal_event));
3302         if(!c->ev) {
3303                 free(c);
3304                 return NULL;
3305         }
3306         c->ev->base = base;
3307         c->fd = -1;
3308         c->buffer = sldns_buffer_new(bufsize);
3309         if(!c->buffer) {
3310                 free(c->ev);
3311                 free(c);
3312                 return NULL;
3313         }
3314         c->timeout = NULL;
3315         c->tcp_is_reading = 0;
3316         c->tcp_byte_count = 0;
3317         c->tcp_timeout_msec = TCP_QUERY_TIMEOUT;
3318         c->tcp_conn_limit = NULL;
3319         c->tcl_addr = NULL;
3320         c->tcp_keepalive = 0;
3321         c->tcp_parent = NULL;
3322         c->max_tcp_count = 0;
3323         c->cur_tcp_count = 0;
3324         c->tcp_handlers = NULL;
3325         c->tcp_free = NULL;
3326         c->type = comm_tcp;
3327         c->tcp_do_close = 0;
3328         c->do_not_close = 0;
3329         c->tcp_do_toggle_rw = 1;
3330         c->tcp_check_nb_connect = 1;
3331 #ifdef USE_MSG_FASTOPEN
3332         c->tcp_do_fastopen = 1;
3333 #endif
3334 #ifdef USE_DNSCRYPT
3335         c->dnscrypt = 0;
3336         c->dnscrypt_buffer = c->buffer;
3337 #endif
3338         c->repinfo.c = c;
3339         c->callback = callback;
3340         c->cb_arg = callback_arg;
3341         evbits = UB_EV_PERSIST | UB_EV_WRITE;
3342         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3343                 comm_point_tcp_handle_callback, c);
3344         if(c->ev->ev == NULL)
3345         {
3346                 log_err("could not baseset tcpout event");
3347                 sldns_buffer_free(c->buffer);
3348                 free(c->ev);
3349                 free(c);
3350                 return NULL;
3351         }
3352
3353         return c;
3354 }
3355
3356 struct comm_point*
3357 comm_point_create_http_out(struct comm_base *base, size_t bufsize,
3358         comm_point_callback_type* callback, void* callback_arg,
3359         sldns_buffer* temp)
3360 {
3361         struct comm_point* c = (struct comm_point*)calloc(1,
3362                 sizeof(struct comm_point));
3363         short evbits;
3364         if(!c)
3365                 return NULL;
3366         c->ev = (struct internal_event*)calloc(1,
3367                 sizeof(struct internal_event));
3368         if(!c->ev) {
3369                 free(c);
3370                 return NULL;
3371         }
3372         c->ev->base = base;
3373         c->fd = -1;
3374         c->buffer = sldns_buffer_new(bufsize);
3375         if(!c->buffer) {
3376                 free(c->ev);
3377                 free(c);
3378                 return NULL;
3379         }
3380         c->timeout = NULL;
3381         c->tcp_is_reading = 0;
3382         c->tcp_byte_count = 0;
3383         c->tcp_parent = NULL;
3384         c->max_tcp_count = 0;
3385         c->cur_tcp_count = 0;
3386         c->tcp_handlers = NULL;
3387         c->tcp_free = NULL;
3388         c->type = comm_http;
3389         c->tcp_do_close = 0;
3390         c->do_not_close = 0;
3391         c->tcp_do_toggle_rw = 1;
3392         c->tcp_check_nb_connect = 1;
3393         c->http_in_headers = 1;
3394         c->http_in_chunk_headers = 0;
3395         c->http_is_chunked = 0;
3396         c->http_temp = temp;
3397 #ifdef USE_MSG_FASTOPEN
3398         c->tcp_do_fastopen = 1;
3399 #endif
3400 #ifdef USE_DNSCRYPT
3401         c->dnscrypt = 0;
3402         c->dnscrypt_buffer = c->buffer;
3403 #endif
3404         c->repinfo.c = c;
3405         c->callback = callback;
3406         c->cb_arg = callback_arg;
3407         evbits = UB_EV_PERSIST | UB_EV_WRITE;
3408         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3409                 comm_point_http_handle_callback, c);
3410         if(c->ev->ev == NULL)
3411         {
3412                 log_err("could not baseset tcpout event");
3413 #ifdef HAVE_SSL
3414                 SSL_free(c->ssl);
3415 #endif
3416                 sldns_buffer_free(c->buffer);
3417                 free(c->ev);
3418                 free(c);
3419                 return NULL;
3420         }
3421
3422         return c;
3423 }
3424
3425 struct comm_point*
3426 comm_point_create_local(struct comm_base *base, int fd, size_t bufsize,
3427         comm_point_callback_type* callback, void* callback_arg)
3428 {
3429         struct comm_point* c = (struct comm_point*)calloc(1,
3430                 sizeof(struct comm_point));
3431         short evbits;
3432         if(!c)
3433                 return NULL;
3434         c->ev = (struct internal_event*)calloc(1,
3435                 sizeof(struct internal_event));
3436         if(!c->ev) {
3437                 free(c);
3438                 return NULL;
3439         }
3440         c->ev->base = base;
3441         c->fd = fd;
3442         c->buffer = sldns_buffer_new(bufsize);
3443         if(!c->buffer) {
3444                 free(c->ev);
3445                 free(c);
3446                 return NULL;
3447         }
3448         c->timeout = NULL;
3449         c->tcp_is_reading = 1;
3450         c->tcp_byte_count = 0;
3451         c->tcp_parent = NULL;
3452         c->max_tcp_count = 0;
3453         c->cur_tcp_count = 0;
3454         c->tcp_handlers = NULL;
3455         c->tcp_free = NULL;
3456         c->type = comm_local;
3457         c->tcp_do_close = 0;
3458         c->do_not_close = 1;
3459         c->tcp_do_toggle_rw = 0;
3460         c->tcp_check_nb_connect = 0;
3461 #ifdef USE_MSG_FASTOPEN
3462         c->tcp_do_fastopen = 0;
3463 #endif
3464 #ifdef USE_DNSCRYPT
3465         c->dnscrypt = 0;
3466         c->dnscrypt_buffer = c->buffer;
3467 #endif
3468         c->callback = callback;
3469         c->cb_arg = callback_arg;
3470         /* ub_event stuff */
3471         evbits = UB_EV_PERSIST | UB_EV_READ;
3472         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3473                 comm_point_local_handle_callback, c);
3474         if(c->ev->ev == NULL) {
3475                 log_err("could not baseset localhdl event");
3476                 free(c->ev);
3477                 free(c);
3478                 return NULL;
3479         }
3480         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3481                 log_err("could not add localhdl event");
3482                 ub_event_free(c->ev->ev);
3483                 free(c->ev);
3484                 free(c);
3485                 return NULL;
3486         }
3487         return c;
3488 }
3489
3490 struct comm_point*
3491 comm_point_create_raw(struct comm_base* base, int fd, int writing,
3492         comm_point_callback_type* callback, void* callback_arg)
3493 {
3494         struct comm_point* c = (struct comm_point*)calloc(1,
3495                 sizeof(struct comm_point));
3496         short evbits;
3497         if(!c)
3498                 return NULL;
3499         c->ev = (struct internal_event*)calloc(1,
3500                 sizeof(struct internal_event));
3501         if(!c->ev) {
3502                 free(c);
3503                 return NULL;
3504         }
3505         c->ev->base = base;
3506         c->fd = fd;
3507         c->buffer = NULL;
3508         c->timeout = NULL;
3509         c->tcp_is_reading = 0;
3510         c->tcp_byte_count = 0;
3511         c->tcp_parent = NULL;
3512         c->max_tcp_count = 0;
3513         c->cur_tcp_count = 0;
3514         c->tcp_handlers = NULL;
3515         c->tcp_free = NULL;
3516         c->type = comm_raw;
3517         c->tcp_do_close = 0;
3518         c->do_not_close = 1;
3519         c->tcp_do_toggle_rw = 0;
3520         c->tcp_check_nb_connect = 0;
3521 #ifdef USE_MSG_FASTOPEN
3522         c->tcp_do_fastopen = 0;
3523 #endif
3524 #ifdef USE_DNSCRYPT
3525         c->dnscrypt = 0;
3526         c->dnscrypt_buffer = c->buffer;
3527 #endif
3528         c->callback = callback;
3529         c->cb_arg = callback_arg;
3530         /* ub_event stuff */
3531         if(writing)
3532                 evbits = UB_EV_PERSIST | UB_EV_WRITE;
3533         else    evbits = UB_EV_PERSIST | UB_EV_READ;
3534         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3535                 comm_point_raw_handle_callback, c);
3536         if(c->ev->ev == NULL) {
3537                 log_err("could not baseset rawhdl event");
3538                 free(c->ev);
3539                 free(c);
3540                 return NULL;
3541         }
3542         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
3543                 log_err("could not add rawhdl event");
3544                 ub_event_free(c->ev->ev);
3545                 free(c->ev);
3546                 free(c);
3547                 return NULL;
3548         }
3549         return c;
3550 }
3551
3552 void
3553 comm_point_close(struct comm_point* c)
3554 {
3555         if(!c)
3556                 return;
3557         if(c->fd != -1) {
3558                 if(ub_event_del(c->ev->ev) != 0) {
3559                         log_err("could not event_del on close");
3560                 }
3561         }
3562         tcl_close_connection(c->tcl_addr);
3563         if(c->tcp_req_info)
3564                 tcp_req_info_clear(c->tcp_req_info);
3565         if(c->h2_session)
3566                 http2_session_server_delete(c->h2_session);
3567
3568         /* close fd after removing from event lists, or epoll.. is messed up */
3569         if(c->fd != -1 && !c->do_not_close) {
3570                 if(c->type == comm_tcp || c->type == comm_http) {
3571                         /* delete sticky events for the fd, it gets closed */
3572                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
3573                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
3574                 }
3575                 verbose(VERB_ALGO, "close fd %d", c->fd);
3576                 sock_close(c->fd);
3577         }
3578         c->fd = -1;
3579 }
3580
3581 void
3582 comm_point_delete(struct comm_point* c)
3583 {
3584         if(!c)
3585                 return;
3586         if((c->type == comm_tcp || c->type == comm_http) && c->ssl) {
3587 #ifdef HAVE_SSL
3588                 SSL_shutdown(c->ssl);
3589                 SSL_free(c->ssl);
3590 #endif
3591         }
3592         if(c->type == comm_http && c->http_endpoint) {
3593                 free(c->http_endpoint);
3594                 c->http_endpoint = NULL;
3595         }
3596         comm_point_close(c);
3597         if(c->tcp_handlers) {
3598                 int i;
3599                 for(i=0; i<c->max_tcp_count; i++)
3600                         comm_point_delete(c->tcp_handlers[i]);
3601                 free(c->tcp_handlers);
3602         }
3603         free(c->timeout);
3604         if(c->type == comm_tcp || c->type == comm_local || c->type == comm_http) {
3605                 sldns_buffer_free(c->buffer);
3606 #ifdef USE_DNSCRYPT
3607                 if(c->dnscrypt && c->dnscrypt_buffer != c->buffer) {
3608                         sldns_buffer_free(c->dnscrypt_buffer);
3609                 }
3610 #endif
3611                 if(c->tcp_req_info) {
3612                         tcp_req_info_delete(c->tcp_req_info);
3613                 }
3614                 if(c->h2_session) {
3615                         http2_session_delete(c->h2_session);
3616                 }
3617         }
3618         ub_event_free(c->ev->ev);
3619         free(c->ev);
3620         free(c);
3621 }
3622
3623 void
3624 comm_point_send_reply(struct comm_reply *repinfo)
3625 {
3626         struct sldns_buffer* buffer;
3627         log_assert(repinfo && repinfo->c);
3628 #ifdef USE_DNSCRYPT
3629         buffer = repinfo->c->dnscrypt_buffer;
3630         if(!dnsc_handle_uncurved_request(repinfo)) {
3631                 return;
3632         }
3633 #else
3634         buffer = repinfo->c->buffer;
3635 #endif
3636         if(repinfo->c->type == comm_udp) {
3637                 if(repinfo->srctype)
3638                         comm_point_send_udp_msg_if(repinfo->c,
3639                         buffer, (struct sockaddr*)&repinfo->addr,
3640                         repinfo->addrlen, repinfo);
3641                 else
3642                         comm_point_send_udp_msg(repinfo->c, buffer,
3643                         (struct sockaddr*)&repinfo->addr, repinfo->addrlen);
3644 #ifdef USE_DNSTAP
3645                 if(repinfo->c->dtenv != NULL &&
3646                    repinfo->c->dtenv->log_client_response_messages)
3647                         dt_msg_send_client_response(repinfo->c->dtenv,
3648                         &repinfo->addr, repinfo->c->type, repinfo->c->buffer);
3649 #endif
3650         } else {
3651 #ifdef USE_DNSTAP
3652                 if(repinfo->c->tcp_parent->dtenv != NULL &&
3653                    repinfo->c->tcp_parent->dtenv->log_client_response_messages)
3654                         dt_msg_send_client_response(repinfo->c->tcp_parent->dtenv,
3655                         &repinfo->addr, repinfo->c->type,
3656                         ( repinfo->c->tcp_req_info
3657                         ? repinfo->c->tcp_req_info->spool_buffer
3658                         : repinfo->c->buffer ));
3659 #endif
3660                 if(repinfo->c->tcp_req_info) {
3661                         tcp_req_info_send_reply(repinfo->c->tcp_req_info);
3662                 } else if(repinfo->c->use_h2) {
3663                         if(!http2_submit_dns_response(repinfo->c->h2_session)) {
3664                                 comm_point_drop_reply(repinfo);
3665                                 return;
3666                         }
3667                         repinfo->c->h2_stream = NULL;
3668                         repinfo->c->tcp_is_reading = 0;
3669                         comm_point_stop_listening(repinfo->c);
3670                         comm_point_start_listening(repinfo->c, -1,
3671                                 repinfo->c->tcp_timeout_msec);
3672                         return;
3673                 } else {
3674                         comm_point_start_listening(repinfo->c, -1,
3675                                 repinfo->c->tcp_timeout_msec);
3676                 }
3677         }
3678 }
3679
3680 void
3681 comm_point_drop_reply(struct comm_reply* repinfo)
3682 {
3683         if(!repinfo)
3684                 return;
3685         log_assert(repinfo->c);
3686         log_assert(repinfo->c->type != comm_tcp_accept);
3687         if(repinfo->c->type == comm_udp)
3688                 return;
3689         if(repinfo->c->tcp_req_info)
3690                 repinfo->c->tcp_req_info->is_drop = 1;
3691         if(repinfo->c->type == comm_http) {
3692                 if(repinfo->c->h2_session) {
3693                         repinfo->c->h2_session->is_drop = 1;
3694                         if(!repinfo->c->h2_session->postpone_drop)
3695                                 reclaim_http_handler(repinfo->c);
3696                         return;
3697                 }
3698                 reclaim_http_handler(repinfo->c);
3699                 return;
3700         }
3701         reclaim_tcp_handler(repinfo->c);
3702 }
3703
3704 void
3705 comm_point_stop_listening(struct comm_point* c)
3706 {
3707         verbose(VERB_ALGO, "comm point stop listening %d", c->fd);
3708         if(ub_event_del(c->ev->ev) != 0) {
3709                 log_err("event_del error to stoplisten");
3710         }
3711 }
3712
3713 void
3714 comm_point_start_listening(struct comm_point* c, int newfd, int msec)
3715 {
3716         verbose(VERB_ALGO, "comm point start listening %d (%d msec)",
3717                 c->fd==-1?newfd:c->fd, msec);
3718         if(c->type == comm_tcp_accept && !c->tcp_free) {
3719                 /* no use to start listening no free slots. */
3720                 return;
3721         }
3722         if(msec != -1 && msec != 0) {
3723                 if(!c->timeout) {
3724                         c->timeout = (struct timeval*)malloc(sizeof(
3725                                 struct timeval));
3726                         if(!c->timeout) {
3727                                 log_err("cpsl: malloc failed. No net read.");
3728                                 return;
3729                         }
3730                 }
3731                 ub_event_add_bits(c->ev->ev, UB_EV_TIMEOUT);
3732 #ifndef S_SPLINT_S /* splint fails on struct timeval. */
3733                 c->timeout->tv_sec = msec/1000;
3734                 c->timeout->tv_usec = (msec%1000)*1000;
3735 #endif /* S_SPLINT_S */
3736         }
3737         if(c->type == comm_tcp || c->type == comm_http) {
3738                 ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
3739                 if(c->tcp_is_reading)
3740                         ub_event_add_bits(c->ev->ev, UB_EV_READ);
3741                 else    ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
3742         }
3743         if(newfd != -1) {
3744                 if(c->fd != -1) {
3745                         sock_close(c->fd);
3746                 }
3747                 c->fd = newfd;
3748                 ub_event_set_fd(c->ev->ev, c->fd);
3749         }
3750         if(ub_event_add(c->ev->ev, msec==0?NULL:c->timeout) != 0) {
3751                 log_err("event_add failed. in cpsl.");
3752         }
3753 }
3754
3755 void comm_point_listen_for_rw(struct comm_point* c, int rd, int wr)
3756 {
3757         verbose(VERB_ALGO, "comm point listen_for_rw %d %d", c->fd, wr);
3758         if(ub_event_del(c->ev->ev) != 0) {
3759                 log_err("event_del error to cplf");
3760         }
3761         ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
3762         if(rd) ub_event_add_bits(c->ev->ev, UB_EV_READ);
3763         if(wr) ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
3764         if(ub_event_add(c->ev->ev, c->timeout) != 0) {
3765                 log_err("event_add failed. in cplf.");
3766         }
3767 }
3768
3769 size_t comm_point_get_mem(struct comm_point* c)
3770 {
3771         size_t s;
3772         if(!c)
3773                 return 0;
3774         s = sizeof(*c) + sizeof(*c->ev);
3775         if(c->timeout)
3776                 s += sizeof(*c->timeout);
3777         if(c->type == comm_tcp || c->type == comm_local) {
3778                 s += sizeof(*c->buffer) + sldns_buffer_capacity(c->buffer);
3779 #ifdef USE_DNSCRYPT
3780                 s += sizeof(*c->dnscrypt_buffer);
3781                 if(c->buffer != c->dnscrypt_buffer) {
3782                         s += sldns_buffer_capacity(c->dnscrypt_buffer);
3783                 }
3784 #endif
3785         }
3786         if(c->type == comm_tcp_accept) {
3787                 int i;
3788                 for(i=0; i<c->max_tcp_count; i++)
3789                         s += comm_point_get_mem(c->tcp_handlers[i]);
3790         }
3791         return s;
3792 }
3793
3794 struct comm_timer*
3795 comm_timer_create(struct comm_base* base, void (*cb)(void*), void* cb_arg)
3796 {
3797         struct internal_timer *tm = (struct internal_timer*)calloc(1,
3798                 sizeof(struct internal_timer));
3799         if(!tm) {
3800                 log_err("malloc failed");
3801                 return NULL;
3802         }
3803         tm->super.ev_timer = tm;
3804         tm->base = base;
3805         tm->super.callback = cb;
3806         tm->super.cb_arg = cb_arg;
3807         tm->ev = ub_event_new(base->eb->base, -1, UB_EV_TIMEOUT,
3808                 comm_timer_callback, &tm->super);
3809         if(tm->ev == NULL) {
3810                 log_err("timer_create: event_base_set failed.");
3811                 free(tm);
3812                 return NULL;
3813         }
3814         return &tm->super;
3815 }
3816
3817 void
3818 comm_timer_disable(struct comm_timer* timer)
3819 {
3820         if(!timer)
3821                 return;
3822         ub_timer_del(timer->ev_timer->ev);
3823         timer->ev_timer->enabled = 0;
3824 }
3825
3826 void
3827 comm_timer_set(struct comm_timer* timer, struct timeval* tv)
3828 {
3829         log_assert(tv);
3830         if(timer->ev_timer->enabled)
3831                 comm_timer_disable(timer);
3832         if(ub_timer_add(timer->ev_timer->ev, timer->ev_timer->base->eb->base,
3833                 comm_timer_callback, timer, tv) != 0)
3834                 log_err("comm_timer_set: evtimer_add failed.");
3835         timer->ev_timer->enabled = 1;
3836 }
3837
3838 void
3839 comm_timer_delete(struct comm_timer* timer)
3840 {
3841         if(!timer)
3842                 return;
3843         comm_timer_disable(timer);
3844         /* Free the sub struct timer->ev_timer derived from the super struct timer.
3845          * i.e. assert(timer == timer->ev_timer)
3846          */
3847         ub_event_free(timer->ev_timer->ev);
3848         free(timer->ev_timer);
3849 }
3850
3851 void
3852 comm_timer_callback(int ATTR_UNUSED(fd), short event, void* arg)
3853 {
3854         struct comm_timer* tm = (struct comm_timer*)arg;
3855         if(!(event&UB_EV_TIMEOUT))
3856                 return;
3857         ub_comm_base_now(tm->ev_timer->base);
3858         tm->ev_timer->enabled = 0;
3859         fptr_ok(fptr_whitelist_comm_timer(tm->callback));
3860         (*tm->callback)(tm->cb_arg);
3861 }
3862
3863 int
3864 comm_timer_is_set(struct comm_timer* timer)
3865 {
3866         return (int)timer->ev_timer->enabled;
3867 }
3868
3869 size_t
3870 comm_timer_get_mem(struct comm_timer* ATTR_UNUSED(timer))
3871 {
3872         return sizeof(struct internal_timer);
3873 }
3874
3875 struct comm_signal*
3876 comm_signal_create(struct comm_base* base,
3877         void (*callback)(int, void*), void* cb_arg)
3878 {
3879         struct comm_signal* com = (struct comm_signal*)malloc(
3880                 sizeof(struct comm_signal));
3881         if(!com) {
3882                 log_err("malloc failed");
3883                 return NULL;
3884         }
3885         com->base = base;
3886         com->callback = callback;
3887         com->cb_arg = cb_arg;
3888         com->ev_signal = NULL;
3889         return com;
3890 }
3891
3892 void
3893 comm_signal_callback(int sig, short event, void* arg)
3894 {
3895         struct comm_signal* comsig = (struct comm_signal*)arg;
3896         if(!(event & UB_EV_SIGNAL))
3897                 return;
3898         ub_comm_base_now(comsig->base);
3899         fptr_ok(fptr_whitelist_comm_signal(comsig->callback));
3900         (*comsig->callback)(sig, comsig->cb_arg);
3901 }
3902
3903 int
3904 comm_signal_bind(struct comm_signal* comsig, int sig)
3905 {
3906         struct internal_signal* entry = (struct internal_signal*)calloc(1,
3907                 sizeof(struct internal_signal));
3908         if(!entry) {
3909                 log_err("malloc failed");
3910                 return 0;
3911         }
3912         log_assert(comsig);
3913         /* add signal event */
3914         entry->ev = ub_signal_new(comsig->base->eb->base, sig,
3915                 comm_signal_callback, comsig);
3916         if(entry->ev == NULL) {
3917                 log_err("Could not create signal event");
3918                 free(entry);
3919                 return 0;
3920         }
3921         if(ub_signal_add(entry->ev, NULL) != 0) {
3922                 log_err("Could not add signal handler");
3923                 ub_event_free(entry->ev);
3924                 free(entry);
3925                 return 0;
3926         }
3927         /* link into list */
3928         entry->next = comsig->ev_signal;
3929         comsig->ev_signal = entry;
3930         return 1;
3931 }
3932
3933 void
3934 comm_signal_delete(struct comm_signal* comsig)
3935 {
3936         struct internal_signal* p, *np;
3937         if(!comsig)
3938                 return;
3939         p=comsig->ev_signal;
3940         while(p) {
3941                 np = p->next;
3942                 ub_signal_del(p->ev);
3943                 ub_event_free(p->ev);
3944                 free(p);
3945                 p = np;
3946         }
3947         free(comsig);
3948 }