contrib/unbound/util/netevent.c

   1 /*
   2  * util/netevent.c - event notification
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains event notification functions.
  40  */
  41 #include "config.h"
  42 #include "util/netevent.h"
  43 #include "util/ub_event.h"
  44 #include "util/log.h"
  45 #include "util/net_help.h"
  46 #include "util/tcp_conn_limit.h"
  47 #include "util/fptr_wlist.h"
  48 #include "util/proxy_protocol.h"
  49 #include "util/timeval_func.h"
  50 #include "sldns/pkthdr.h"
  51 #include "sldns/sbuffer.h"
  52 #include "sldns/str2wire.h"
  53 #include "dnstap/dnstap.h"
  54 #include "dnscrypt/dnscrypt.h"
  55 #include "services/listen_dnsport.h"
  56 #ifdef HAVE_SYS_TYPES_H
  57 #include <sys/types.h>
  58 #endif
  59 #ifdef HAVE_SYS_SOCKET_H
  60 #include <sys/socket.h>
  61 #endif
  62 #ifdef HAVE_NETDB_H
  63 #include <netdb.h>
  64 #endif
  65 #ifdef HAVE_POLL_H
  66 #include <poll.h>
  67 #endif
  68
  69 #ifdef HAVE_OPENSSL_SSL_H
  70 #include <openssl/ssl.h>
  71 #endif
  72 #ifdef HAVE_OPENSSL_ERR_H
  73 #include <openssl/err.h>
  74 #endif
  75 #ifdef HAVE_LINUX_NET_TSTAMP_H
  76 #include <linux/net_tstamp.h>
  77 #endif
  78 /* -------- Start of local definitions -------- */
  79 /** if CMSG_ALIGN is not defined on this platform, a workaround */
  80 #ifndef CMSG_ALIGN
  81 #  ifdef __CMSG_ALIGN
  82 #    define CMSG_ALIGN(n) __CMSG_ALIGN(n)
  83 #  elif defined(CMSG_DATA_ALIGN)
  84 #    define CMSG_ALIGN _CMSG_DATA_ALIGN
  85 #  else
  86 #    define CMSG_ALIGN(len) (((len)+sizeof(long)-1) & ~(sizeof(long)-1))
  87 #  endif
  88 #endif
  89
  90 /** if CMSG_LEN is not defined on this platform, a workaround */
  91 #ifndef CMSG_LEN
  92 #  define CMSG_LEN(len) (CMSG_ALIGN(sizeof(struct cmsghdr))+(len))
  93 #endif
  94
  95 /** if CMSG_SPACE is not defined on this platform, a workaround */
  96 #ifndef CMSG_SPACE
  97 #  ifdef _CMSG_HDR_ALIGN
  98 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+_CMSG_HDR_ALIGN(sizeof(struct cmsghdr)))
  99 #  else
 100 #    define CMSG_SPACE(l) (CMSG_ALIGN(l)+CMSG_ALIGN(sizeof(struct cmsghdr)))
 101 #  endif
 102 #endif
 103
 104 /** The TCP writing query timeout in milliseconds */
 105 #define TCP_QUERY_TIMEOUT 120000
 106 /** The minimum actual TCP timeout to use, regardless of what we advertise,
 107  * in msec */
 108 #define TCP_QUERY_TIMEOUT_MINIMUM 200
 109
 110 #ifndef NONBLOCKING_IS_BROKEN
 111 /** number of UDP reads to perform per read indication from select */
 112 #define NUM_UDP_PER_SELECT 100
 113 #else
 114 #define NUM_UDP_PER_SELECT 1
 115 #endif
 116
 117 /** timeout in millisec to wait for write to unblock, packets dropped after.*/
 118 #define SEND_BLOCKED_WAIT_TIMEOUT 200
 119
 120 /** Let's make timestamping code cleaner and redefine SO_TIMESTAMP* */
 121 #ifndef SO_TIMESTAMP
 122 #define SO_TIMESTAMP 29
 123 #endif
 124 #ifndef SO_TIMESTAMPNS
 125 #define SO_TIMESTAMPNS 35
 126 #endif
 127 #ifndef SO_TIMESTAMPING
 128 #define SO_TIMESTAMPING 37
 129 #endif
 130 /**
 131  * The internal event structure for keeping ub_event info for the event.
 132  * Possibly other structures (list, tree) this is part of.
 133  */
 134 struct internal_event {
 135         /** the comm base */
 136         struct comm_base* base;
 137         /** ub_event event type */
 138         struct ub_event* ev;
 139 };
 140
 141 /**
 142  * Internal base structure, so that every thread has its own events.
 143  */
 144 struct internal_base {
 145         /** ub_event event_base type. */
 146         struct ub_event_base* base;
 147         /** seconds time pointer points here */
 148         time_t secs;
 149         /** timeval with current time */
 150         struct timeval now;
 151         /** the event used for slow_accept timeouts */
 152         struct ub_event* slow_accept;
 153         /** true if slow_accept is enabled */
 154         int slow_accept_enabled;
 155         /** last log time for slow logging of file descriptor errors */
 156         time_t last_slow_log;
 157         /** last log time for slow logging of write wait failures */
 158         time_t last_writewait_log;
 159 };
 160
 161 /**
 162  * Internal timer structure, to store timer event in.
 163  */
 164 struct internal_timer {
 165         /** the super struct from which derived */
 166         struct comm_timer super;
 167         /** the comm base */
 168         struct comm_base* base;
 169         /** ub_event event type */
 170         struct ub_event* ev;
 171         /** is timer enabled */
 172         uint8_t enabled;
 173 };
 174
 175 /**
 176  * Internal signal structure, to store signal event in.
 177  */
 178 struct internal_signal {
 179         /** ub_event event type */
 180         struct ub_event* ev;
 181         /** next in signal list */
 182         struct internal_signal* next;
 183 };
 184
 185 /** create a tcp handler with a parent */
 186 static struct comm_point* comm_point_create_tcp_handler(
 187         struct comm_base *base, struct comm_point* parent, size_t bufsize,
 188         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
 189         void* callback_arg, struct unbound_socket* socket);
 190
 191 /* -------- End of local definitions -------- */
 192
 193 struct comm_base*
 194 comm_base_create(int sigs)
 195 {
 196         struct comm_base* b = (struct comm_base*)calloc(1,
 197                 sizeof(struct comm_base));
 198         const char *evnm="event", *evsys="", *evmethod="";
 199
 200         if(!b)
 201                 return NULL;
 202         b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
 203         if(!b->eb) {
 204                 free(b);
 205                 return NULL;
 206         }
 207         b->eb->base = ub_default_event_base(sigs, &b->eb->secs, &b->eb->now);
 208         if(!b->eb->base) {
 209                 free(b->eb);
 210                 free(b);
 211                 return NULL;
 212         }
 213         ub_comm_base_now(b);
 214         ub_get_event_sys(b->eb->base, &evnm, &evsys, &evmethod);
 215         verbose(VERB_ALGO, "%s %s uses %s method.", evnm, evsys, evmethod);
 216         return b;
 217 }
 218
 219 struct comm_base*
 220 comm_base_create_event(struct ub_event_base* base)
 221 {
 222         struct comm_base* b = (struct comm_base*)calloc(1,
 223                 sizeof(struct comm_base));
 224         if(!b)
 225                 return NULL;
 226         b->eb = (struct internal_base*)calloc(1, sizeof(struct internal_base));
 227         if(!b->eb) {
 228                 free(b);
 229                 return NULL;
 230         }
 231         b->eb->base = base;
 232         ub_comm_base_now(b);
 233         return b;
 234 }
 235
 236 void
 237 comm_base_delete(struct comm_base* b)
 238 {
 239         if(!b)
 240                 return;
 241         if(b->eb->slow_accept_enabled) {
 242                 if(ub_event_del(b->eb->slow_accept) != 0) {
 243                         log_err("could not event_del slow_accept");
 244                 }
 245                 ub_event_free(b->eb->slow_accept);
 246         }
 247         ub_event_base_free(b->eb->base);
 248         b->eb->base = NULL;
 249         free(b->eb);
 250         free(b);
 251 }
 252
 253 void
 254 comm_base_delete_no_base(struct comm_base* b)
 255 {
 256         if(!b)
 257                 return;
 258         if(b->eb->slow_accept_enabled) {
 259                 if(ub_event_del(b->eb->slow_accept) != 0) {
 260                         log_err("could not event_del slow_accept");
 261                 }
 262                 ub_event_free(b->eb->slow_accept);
 263         }
 264         b->eb->base = NULL;
 265         free(b->eb);
 266         free(b);
 267 }
 268
 269 void
 270 comm_base_timept(struct comm_base* b, time_t** tt, struct timeval** tv)
 271 {
 272         *tt = &b->eb->secs;
 273         *tv = &b->eb->now;
 274 }
 275
 276 void
 277 comm_base_dispatch(struct comm_base* b)
 278 {
 279         int retval;
 280         retval = ub_event_base_dispatch(b->eb->base);
 281         if(retval < 0) {
 282                 fatal_exit("event_dispatch returned error %d, "
 283                         "errno is %s", retval, strerror(errno));
 284         }
 285 }
 286
 287 void comm_base_exit(struct comm_base* b)
 288 {
 289         if(ub_event_base_loopexit(b->eb->base) != 0) {
 290                 log_err("Could not loopexit");
 291         }
 292 }
 293
 294 void comm_base_set_slow_accept_handlers(struct comm_base* b,
 295         void (*stop_acc)(void*), void (*start_acc)(void*), void* arg)
 296 {
 297         b->stop_accept = stop_acc;
 298         b->start_accept = start_acc;
 299         b->cb_arg = arg;
 300 }
 301
 302 struct ub_event_base* comm_base_internal(struct comm_base* b)
 303 {
 304         return b->eb->base;
 305 }
 306
 307 /** see if errno for udp has to be logged or not uses globals */
 308 static int
 309 udp_send_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
 310 {
 311         /* do not log transient errors (unless high verbosity) */
 312 #if defined(ENETUNREACH) || defined(EHOSTDOWN) || defined(EHOSTUNREACH) || defined(ENETDOWN)
 313         switch(errno) {
 314 #  ifdef ENETUNREACH
 315                 case ENETUNREACH:
 316 #  endif
 317 #  ifdef EHOSTDOWN
 318                 case EHOSTDOWN:
 319 #  endif
 320 #  ifdef EHOSTUNREACH
 321                 case EHOSTUNREACH:
 322 #  endif
 323 #  ifdef ENETDOWN
 324                 case ENETDOWN:
 325 #  endif
 326                 case EPERM:
 327                 case EACCES:
 328                         if(verbosity < VERB_ALGO)
 329                                 return 0;
 330                 default:
 331                         break;
 332         }
 333 #endif
 334         /* permission denied is gotten for every send if the
 335          * network is disconnected (on some OS), squelch it */
 336         if( ((errno == EPERM)
 337 #  ifdef EADDRNOTAVAIL
 338                 /* 'Cannot assign requested address' also when disconnected */
 339                 || (errno == EADDRNOTAVAIL)
 340 #  endif
 341                 ) && verbosity < VERB_ALGO)
 342                 return 0;
 343 #  ifdef EADDRINUSE
 344         /* If SO_REUSEADDR is set, we could try to connect to the same server
 345          * from the same source port twice. */
 346         if(errno == EADDRINUSE && verbosity < VERB_DETAIL)
 347                 return 0;
 348 #  endif
 349         /* squelch errors where people deploy AAAA ::ffff:bla for
 350          * authority servers, which we try for intranets. */
 351         if(errno == EINVAL && addr_is_ip4mapped(
 352                 (struct sockaddr_storage*)addr, addrlen) &&
 353                 verbosity < VERB_DETAIL)
 354                 return 0;
 355         /* SO_BROADCAST sockopt can give access to 255.255.255.255,
 356          * but a dns cache does not need it. */
 357         if(errno == EACCES && addr_is_broadcast(
 358                 (struct sockaddr_storage*)addr, addrlen) &&
 359                 verbosity < VERB_DETAIL)
 360                 return 0;
 361         return 1;
 362 }
 363
 364 int tcp_connect_errno_needs_log(struct sockaddr* addr, socklen_t addrlen)
 365 {
 366         return udp_send_errno_needs_log(addr, addrlen);
 367 }
 368
 369 /* send a UDP reply */
 370 int
 371 comm_point_send_udp_msg(struct comm_point *c, sldns_buffer* packet,
 372         struct sockaddr* addr, socklen_t addrlen, int is_connected)
 373 {
 374         ssize_t sent;
 375         log_assert(c->fd != -1);
 376 #ifdef UNBOUND_DEBUG
 377         if(sldns_buffer_remaining(packet) == 0)
 378                 log_err("error: send empty UDP packet");
 379 #endif
 380         log_assert(addr && addrlen > 0);
 381         if(!is_connected) {
 382                 sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
 383                         sldns_buffer_remaining(packet), 0,
 384                         addr, addrlen);
 385         } else {
 386                 sent = send(c->fd, (void*)sldns_buffer_begin(packet),
 387                         sldns_buffer_remaining(packet), 0);
 388         }
 389         if(sent == -1) {
 390                 /* try again and block, waiting for IO to complete,
 391                  * we want to send the answer, and we will wait for
 392                  * the ethernet interface buffer to have space. */
 393 #ifndef USE_WINSOCK
 394                 if(errno == EAGAIN || errno == EINTR ||
 395 #  ifdef EWOULDBLOCK
 396                         errno == EWOULDBLOCK ||
 397 #  endif
 398                         errno == ENOBUFS) {
 399 #else
 400                 if(WSAGetLastError() == WSAEINPROGRESS ||
 401                         WSAGetLastError() == WSAEINTR ||
 402                         WSAGetLastError() == WSAENOBUFS ||
 403                         WSAGetLastError() == WSAEWOULDBLOCK) {
 404 #endif
 405                         /* if we set the fd blocking, other threads suddenly
 406                          * have a blocking fd that they operate on */
 407                         while(sent == -1 && (
 408 #ifndef USE_WINSOCK
 409                                 errno == EAGAIN || errno == EINTR ||
 410 #  ifdef EWOULDBLOCK
 411                                 errno == EWOULDBLOCK ||
 412 #  endif
 413                                 errno == ENOBUFS
 414 #else
 415                                 WSAGetLastError() == WSAEINPROGRESS ||
 416                                 WSAGetLastError() == WSAEINTR ||
 417                                 WSAGetLastError() == WSAENOBUFS ||
 418                                 WSAGetLastError() == WSAEWOULDBLOCK
 419 #endif
 420                         )) {
 421 #if defined(HAVE_POLL) || defined(USE_WINSOCK)
 422                                 struct pollfd p;
 423                                 int pret;
 424                                 memset(&p, 0, sizeof(p));
 425                                 p.fd = c->fd;
 426                                 p.events = POLLOUT | POLLERR | POLLHUP;
 427 #  ifndef USE_WINSOCK
 428                                 pret = poll(&p, 1, SEND_BLOCKED_WAIT_TIMEOUT);
 429 #  else
 430                                 pret = WSAPoll(&p, 1,
 431                                         SEND_BLOCKED_WAIT_TIMEOUT);
 432 #  endif
 433                                 if(pret == 0) {
 434                                         /* timer expired */
 435                                         struct comm_base* b = c->ev->base;
 436                                         if(b->eb->last_writewait_log+SLOW_LOG_TIME <=
 437                                                 b->eb->secs) {
 438                                                 b->eb->last_writewait_log = b->eb->secs;
 439                                                 verbose(VERB_OPS, "send udp blocked "
 440                                                         "for long, dropping packet.");
 441                                         }
 442                                         return 0;
 443                                 } else if(pret < 0 &&
 444 #ifndef USE_WINSOCK
 445                                         errno != EAGAIN && errno != EINTR &&
 446 #  ifdef EWOULDBLOCK
 447                                         errno != EWOULDBLOCK &&
 448 #  endif
 449                                         errno != ENOBUFS
 450 #else
 451                                         WSAGetLastError() != WSAEINPROGRESS &&
 452                                         WSAGetLastError() != WSAEINTR &&
 453                                         WSAGetLastError() != WSAENOBUFS &&
 454                                         WSAGetLastError() != WSAEWOULDBLOCK
 455 #endif
 456                                         ) {
 457                                         log_err("poll udp out failed: %s",
 458                                                 sock_strerror(errno));
 459                                         return 0;
 460                                 }
 461 #endif /* defined(HAVE_POLL) || defined(USE_WINSOCK) */
 462                                 if (!is_connected) {
 463                                         sent = sendto(c->fd, (void*)sldns_buffer_begin(packet),
 464                                                 sldns_buffer_remaining(packet), 0,
 465                                                 addr, addrlen);
 466                                 } else {
 467                                         sent = send(c->fd, (void*)sldns_buffer_begin(packet),
 468                                                 sldns_buffer_remaining(packet), 0);
 469                                 }
 470                         }
 471                 }
 472         }
 473         if(sent == -1) {
 474                 if(!udp_send_errno_needs_log(addr, addrlen))
 475                         return 0;
 476                 if (!is_connected) {
 477                         verbose(VERB_OPS, "sendto failed: %s", sock_strerror(errno));
 478                 } else {
 479                         verbose(VERB_OPS, "send failed: %s", sock_strerror(errno));
 480                 }
 481                 if(addr)
 482                         log_addr(VERB_OPS, "remote address is",
 483                                 (struct sockaddr_storage*)addr, addrlen);
 484                 return 0;
 485         } else if((size_t)sent != sldns_buffer_remaining(packet)) {
 486                 log_err("sent %d in place of %d bytes",
 487                         (int)sent, (int)sldns_buffer_remaining(packet));
 488                 return 0;
 489         }
 490         return 1;
 491 }
 492
 493 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && (defined(HAVE_RECVMSG) || defined(HAVE_SENDMSG))
 494 /** print debug ancillary info */
 495 static void p_ancil(const char* str, struct comm_reply* r)
 496 {
 497         if(r->srctype != 4 && r->srctype != 6) {
 498                 log_info("%s: unknown srctype %d", str, r->srctype);
 499                 return;
 500         }
 501
 502         if(r->srctype == 6) {
 503 #ifdef IPV6_PKTINFO
 504                 char buf[1024];
 505                 if(inet_ntop(AF_INET6, &r->pktinfo.v6info.ipi6_addr,
 506                         buf, (socklen_t)sizeof(buf)) == 0) {
 507                         (void)strlcpy(buf, "(inet_ntop error)", sizeof(buf));
 508                 }
 509                 buf[sizeof(buf)-1]=0;
 510                 log_info("%s: %s %d", str, buf, r->pktinfo.v6info.ipi6_ifindex);
 511 #endif
 512         } else if(r->srctype == 4) {
 513 #ifdef IP_PKTINFO
 514                 char buf1[1024], buf2[1024];
 515                 if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_addr,
 516                         buf1, (socklen_t)sizeof(buf1)) == 0) {
 517                         (void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
 518                 }
 519                 buf1[sizeof(buf1)-1]=0;
 520 #ifdef HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST
 521                 if(inet_ntop(AF_INET, &r->pktinfo.v4info.ipi_spec_dst,
 522                         buf2, (socklen_t)sizeof(buf2)) == 0) {
 523                         (void)strlcpy(buf2, "(inet_ntop error)", sizeof(buf2));
 524                 }
 525                 buf2[sizeof(buf2)-1]=0;
 526 #else
 527                 buf2[0]=0;
 528 #endif
 529                 log_info("%s: %d %s %s", str, r->pktinfo.v4info.ipi_ifindex,
 530                         buf1, buf2);
 531 #elif defined(IP_RECVDSTADDR)
 532                 char buf1[1024];
 533                 if(inet_ntop(AF_INET, &r->pktinfo.v4addr,
 534                         buf1, (socklen_t)sizeof(buf1)) == 0) {
 535                         (void)strlcpy(buf1, "(inet_ntop error)", sizeof(buf1));
 536                 }
 537                 buf1[sizeof(buf1)-1]=0;
 538                 log_info("%s: %s", str, buf1);
 539 #endif /* IP_PKTINFO or PI_RECVDSTDADDR */
 540         }
 541 }
 542 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG||HAVE_SENDMSG */
 543
 544 /** send a UDP reply over specified interface*/
 545 static int
 546 comm_point_send_udp_msg_if(struct comm_point *c, sldns_buffer* packet,
 547         struct sockaddr* addr, socklen_t addrlen, struct comm_reply* r)
 548 {
 549 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_SENDMSG)
 550         ssize_t sent;
 551         struct msghdr msg;
 552         struct iovec iov[1];
 553         union {
 554                 struct cmsghdr hdr;
 555                 char buf[256];
 556         } control;
 557 #ifndef S_SPLINT_S
 558         struct cmsghdr *cmsg;
 559 #endif /* S_SPLINT_S */
 560
 561         log_assert(c->fd != -1);
 562 #ifdef UNBOUND_DEBUG
 563         if(sldns_buffer_remaining(packet) == 0)
 564                 log_err("error: send empty UDP packet");
 565 #endif
 566         log_assert(addr && addrlen > 0);
 567
 568         msg.msg_name = addr;
 569         msg.msg_namelen = addrlen;
 570         iov[0].iov_base = sldns_buffer_begin(packet);
 571         iov[0].iov_len = sldns_buffer_remaining(packet);
 572         msg.msg_iov = iov;
 573         msg.msg_iovlen = 1;
 574         msg.msg_control = control.buf;
 575 #ifndef S_SPLINT_S
 576         msg.msg_controllen = sizeof(control.buf);
 577 #endif /* S_SPLINT_S */
 578         msg.msg_flags = 0;
 579
 580 #ifndef S_SPLINT_S
 581         cmsg = CMSG_FIRSTHDR(&msg);
 582         if(r->srctype == 4) {
 583 #ifdef IP_PKTINFO
 584                 void* cmsg_data;
 585                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo));
 586                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 587                 cmsg->cmsg_level = IPPROTO_IP;
 588                 cmsg->cmsg_type = IP_PKTINFO;
 589                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v4info,
 590                         sizeof(struct in_pktinfo));
 591                 /* unset the ifindex to not bypass the routing tables */
 592                 cmsg_data = CMSG_DATA(cmsg);
 593                 ((struct in_pktinfo *) cmsg_data)->ipi_ifindex = 0;
 594                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
 595                 /* zero the padding bytes inserted by the CMSG_LEN */
 596                 if(sizeof(struct in_pktinfo) < cmsg->cmsg_len)
 597                         memset(((uint8_t*)(CMSG_DATA(cmsg))) +
 598                                 sizeof(struct in_pktinfo), 0, cmsg->cmsg_len
 599                                 - sizeof(struct in_pktinfo));
 600 #elif defined(IP_SENDSRCADDR)
 601                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in_addr));
 602                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 603                 cmsg->cmsg_level = IPPROTO_IP;
 604                 cmsg->cmsg_type = IP_SENDSRCADDR;
 605                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v4addr,
 606                         sizeof(struct in_addr));
 607                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_addr));
 608                 /* zero the padding bytes inserted by the CMSG_LEN */
 609                 if(sizeof(struct in_addr) < cmsg->cmsg_len)
 610                         memset(((uint8_t*)(CMSG_DATA(cmsg))) +
 611                                 sizeof(struct in_addr), 0, cmsg->cmsg_len
 612                                 - sizeof(struct in_addr));
 613 #else
 614                 verbose(VERB_ALGO, "no IP_PKTINFO or IP_SENDSRCADDR");
 615                 msg.msg_control = NULL;
 616 #endif /* IP_PKTINFO or IP_SENDSRCADDR */
 617         } else if(r->srctype == 6) {
 618                 void* cmsg_data;
 619                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
 620                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 621                 cmsg->cmsg_level = IPPROTO_IPV6;
 622                 cmsg->cmsg_type = IPV6_PKTINFO;
 623                 memmove(CMSG_DATA(cmsg), &r->pktinfo.v6info,
 624                         sizeof(struct in6_pktinfo));
 625                 /* unset the ifindex to not bypass the routing tables */
 626                 cmsg_data = CMSG_DATA(cmsg);
 627                 ((struct in6_pktinfo *) cmsg_data)->ipi6_ifindex = 0;
 628                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
 629                 /* zero the padding bytes inserted by the CMSG_LEN */
 630                 if(sizeof(struct in6_pktinfo) < cmsg->cmsg_len)
 631                         memset(((uint8_t*)(CMSG_DATA(cmsg))) +
 632                                 sizeof(struct in6_pktinfo), 0, cmsg->cmsg_len
 633                                 - sizeof(struct in6_pktinfo));
 634         } else {
 635                 /* try to pass all 0 to use default route */
 636                 msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo));
 637                 log_assert(msg.msg_controllen <= sizeof(control.buf));
 638                 cmsg->cmsg_level = IPPROTO_IPV6;
 639                 cmsg->cmsg_type = IPV6_PKTINFO;
 640                 memset(CMSG_DATA(cmsg), 0, sizeof(struct in6_pktinfo));
 641                 cmsg->cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo));
 642                 /* zero the padding bytes inserted by the CMSG_LEN */
 643                 if(sizeof(struct in6_pktinfo) < cmsg->cmsg_len)
 644                         memset(((uint8_t*)(CMSG_DATA(cmsg))) +
 645                                 sizeof(struct in6_pktinfo), 0, cmsg->cmsg_len
 646                                 - sizeof(struct in6_pktinfo));
 647         }
 648 #endif /* S_SPLINT_S */
 649         if(verbosity >= VERB_ALGO && r->srctype != 0)
 650                 p_ancil("send_udp over interface", r);
 651         sent = sendmsg(c->fd, &msg, 0);
 652         if(sent == -1) {
 653                 /* try again and block, waiting for IO to complete,
 654                  * we want to send the answer, and we will wait for
 655                  * the ethernet interface buffer to have space. */
 656 #ifndef USE_WINSOCK
 657                 if(errno == EAGAIN || errno == EINTR ||
 658 #  ifdef EWOULDBLOCK
 659                         errno == EWOULDBLOCK ||
 660 #  endif
 661                         errno == ENOBUFS) {
 662 #else
 663                 if(WSAGetLastError() == WSAEINPROGRESS ||
 664                         WSAGetLastError() == WSAEINTR ||
 665                         WSAGetLastError() == WSAENOBUFS ||
 666                         WSAGetLastError() == WSAEWOULDBLOCK) {
 667 #endif
 668                         while(sent == -1 && (
 669 #ifndef USE_WINSOCK
 670                                 errno == EAGAIN || errno == EINTR ||
 671 #  ifdef EWOULDBLOCK
 672                                 errno == EWOULDBLOCK ||
 673 #  endif
 674                                 errno == ENOBUFS
 675 #else
 676                                 WSAGetLastError() == WSAEINPROGRESS ||
 677                                 WSAGetLastError() == WSAEINTR ||
 678                                 WSAGetLastError() == WSAENOBUFS ||
 679                                 WSAGetLastError() == WSAEWOULDBLOCK
 680 #endif
 681                         )) {
 682 #if defined(HAVE_POLL) || defined(USE_WINSOCK)
 683                                 struct pollfd p;
 684                                 int pret;
 685                                 memset(&p, 0, sizeof(p));
 686                                 p.fd = c->fd;
 687                                 p.events = POLLOUT | POLLERR | POLLHUP;
 688 #  ifndef USE_WINSOCK
 689                                 pret = poll(&p, 1, SEND_BLOCKED_WAIT_TIMEOUT);
 690 #  else
 691                                 pret = WSAPoll(&p, 1,
 692                                         SEND_BLOCKED_WAIT_TIMEOUT);
 693 #  endif
 694                                 if(pret == 0) {
 695                                         /* timer expired */
 696                                         struct comm_base* b = c->ev->base;
 697                                         if(b->eb->last_writewait_log+SLOW_LOG_TIME <=
 698                                                 b->eb->secs) {
 699                                                 b->eb->last_writewait_log = b->eb->secs;
 700                                                 verbose(VERB_OPS, "send udp blocked "
 701                                                         "for long, dropping packet.");
 702                                         }
 703                                         return 0;
 704                                 } else if(pret < 0 &&
 705 #ifndef USE_WINSOCK
 706                                         errno != EAGAIN && errno != EINTR &&
 707 #  ifdef EWOULDBLOCK
 708                                         errno != EWOULDBLOCK &&
 709 #  endif
 710                                         errno != ENOBUFS
 711 #else
 712                                         WSAGetLastError() != WSAEINPROGRESS &&
 713                                         WSAGetLastError() != WSAEINTR &&
 714                                         WSAGetLastError() != WSAENOBUFS &&
 715                                         WSAGetLastError() != WSAEWOULDBLOCK
 716 #endif
 717                                         ) {
 718                                         log_err("poll udp out failed: %s",
 719                                                 sock_strerror(errno));
 720                                         return 0;
 721                                 }
 722 #endif /* defined(HAVE_POLL) || defined(USE_WINSOCK) */
 723                                 sent = sendmsg(c->fd, &msg, 0);
 724                         }
 725                 }
 726         }
 727         if(sent == -1) {
 728                 if(!udp_send_errno_needs_log(addr, addrlen))
 729                         return 0;
 730                 verbose(VERB_OPS, "sendmsg failed: %s", strerror(errno));
 731                 log_addr(VERB_OPS, "remote address is",
 732                         (struct sockaddr_storage*)addr, addrlen);
 733 #ifdef __NetBSD__
 734                 /* netbsd 7 has IP_PKTINFO for recv but not send */
 735                 if(errno == EINVAL && r->srctype == 4)
 736                         log_err("sendmsg: No support for sendmsg(IP_PKTINFO). "
 737                                 "Please disable interface-automatic");
 738 #endif
 739                 return 0;
 740         } else if((size_t)sent != sldns_buffer_remaining(packet)) {
 741                 log_err("sent %d in place of %d bytes",
 742                         (int)sent, (int)sldns_buffer_remaining(packet));
 743                 return 0;
 744         }
 745         return 1;
 746 #else
 747         (void)c;
 748         (void)packet;
 749         (void)addr;
 750         (void)addrlen;
 751         (void)r;
 752         log_err("sendmsg: IPV6_PKTINFO not supported");
 753         return 0;
 754 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_SENDMSG */
 755 }
 756
 757 /** return true is UDP receive error needs to be logged */
 758 static int udp_recv_needs_log(int err)
 759 {
 760         switch(err) {
 761         case EACCES: /* some hosts send ICMP 'Permission Denied' */
 762 #ifndef USE_WINSOCK
 763         case ECONNREFUSED:
 764 #  ifdef ENETUNREACH
 765         case ENETUNREACH:
 766 #  endif
 767 #  ifdef EHOSTDOWN
 768         case EHOSTDOWN:
 769 #  endif
 770 #  ifdef EHOSTUNREACH
 771         case EHOSTUNREACH:
 772 #  endif
 773 #  ifdef ENETDOWN
 774         case ENETDOWN:
 775 #  endif
 776 #else /* USE_WINSOCK */
 777         case WSAECONNREFUSED:
 778         case WSAENETUNREACH:
 779         case WSAEHOSTDOWN:
 780         case WSAEHOSTUNREACH:
 781         case WSAENETDOWN:
 782 #endif
 783                 if(verbosity >= VERB_ALGO)
 784                         return 1;
 785                 return 0;
 786         default:
 787                 break;
 788         }
 789         return 1;
 790 }
 791
 792 /** Parses the PROXYv2 header from buf and updates the comm_reply struct.
 793  *  Returns 1 on success, 0 on failure. */
 794 static int consume_pp2_header(struct sldns_buffer* buf, struct comm_reply* rep,
 795         int stream) {
 796         size_t size;
 797         struct pp2_header *header = pp2_read_header(buf);
 798         if(header == NULL) return 0;
 799         size = PP2_HEADER_SIZE + ntohs(header->len);
 800         if((header->ver_cmd & 0xF) == PP2_CMD_LOCAL) {
 801                 /* A connection from the proxy itself.
 802                  * No need to do anything with addresses. */
 803                 goto done;
 804         }
 805         if(header->fam_prot == 0x00) {
 806                 /* Unspecified family and protocol. This could be used for
 807                  * health checks by proxies.
 808                  * No need to do anything with addresses. */
 809                 goto done;
 810         }
 811         /* Read the proxied address */
 812         switch(header->fam_prot) {
 813                 case 0x11: /* AF_INET|STREAM */
 814                 case 0x12: /* AF_INET|DGRAM */
 815                         {
 816                         struct sockaddr_in* addr =
 817                                 (struct sockaddr_in*)&rep->client_addr;
 818                         addr->sin_family = AF_INET;
 819                         addr->sin_addr.s_addr = header->addr.addr4.src_addr;
 820                         addr->sin_port = header->addr.addr4.src_port;
 821                         rep->client_addrlen = (socklen_t)sizeof(struct sockaddr_in);
 822                         }
 823                         /* Ignore the destination address; it should be us. */
 824                         break;
 825                 case 0x21: /* AF_INET6|STREAM */
 826                 case 0x22: /* AF_INET6|DGRAM */
 827                         {
 828                         struct sockaddr_in6* addr =
 829                                 (struct sockaddr_in6*)&rep->client_addr;
 830                         memset(addr, 0, sizeof(*addr));
 831                         addr->sin6_family = AF_INET6;
 832                         memcpy(&addr->sin6_addr,
 833                                 header->addr.addr6.src_addr, 16);
 834                         addr->sin6_port = header->addr.addr6.src_port;
 835                         rep->client_addrlen = (socklen_t)sizeof(struct sockaddr_in6);
 836                         }
 837                         /* Ignore the destination address; it should be us. */
 838                         break;
 839         }
 840         rep->is_proxied = 1;
 841 done:
 842         if(!stream) {
 843                 /* We are reading a whole packet;
 844                  * Move the rest of the data to overwrite the PROXYv2 header */
 845                 /* XXX can we do better to avoid memmove? */
 846                 memmove(header, ((char*)header)+size,
 847                         sldns_buffer_limit(buf)-size);
 848                 sldns_buffer_set_limit(buf, sldns_buffer_limit(buf)-size);
 849         }
 850         return 1;
 851 }
 852
 853 void
 854 comm_point_udp_ancil_callback(int fd, short event, void* arg)
 855 {
 856 #if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_RECVMSG)
 857         struct comm_reply rep;
 858         struct msghdr msg;
 859         struct iovec iov[1];
 860         ssize_t rcv;
 861         union {
 862                 struct cmsghdr hdr;
 863                 char buf[256];
 864         } ancil;
 865         int i;
 866 #ifndef S_SPLINT_S
 867         struct cmsghdr* cmsg;
 868 #endif /* S_SPLINT_S */
 869 #ifdef HAVE_LINUX_NET_TSTAMP_H
 870         struct timespec *ts;
 871 #endif /* HAVE_LINUX_NET_TSTAMP_H */
 872
 873         rep.c = (struct comm_point*)arg;
 874         log_assert(rep.c->type == comm_udp);
 875
 876         if(!(event&UB_EV_READ))
 877                 return;
 878         log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
 879         ub_comm_base_now(rep.c->ev->base);
 880         for(i=0; i<NUM_UDP_PER_SELECT; i++) {
 881                 sldns_buffer_clear(rep.c->buffer);
 882                 timeval_clear(&rep.c->recv_tv);
 883                 rep.remote_addrlen = (socklen_t)sizeof(rep.remote_addr);
 884                 log_assert(fd != -1);
 885                 log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
 886                 msg.msg_name = &rep.remote_addr;
 887                 msg.msg_namelen = (socklen_t)sizeof(rep.remote_addr);
 888                 iov[0].iov_base = sldns_buffer_begin(rep.c->buffer);
 889                 iov[0].iov_len = sldns_buffer_remaining(rep.c->buffer);
 890                 msg.msg_iov = iov;
 891                 msg.msg_iovlen = 1;
 892                 msg.msg_control = ancil.buf;
 893 #ifndef S_SPLINT_S
 894                 msg.msg_controllen = sizeof(ancil.buf);
 895 #endif /* S_SPLINT_S */
 896                 msg.msg_flags = 0;
 897                 rcv = recvmsg(fd, &msg, MSG_DONTWAIT);
 898                 if(rcv == -1) {
 899                         if(errno != EAGAIN && errno != EINTR
 900                                 && udp_recv_needs_log(errno)) {
 901                                 log_err("recvmsg failed: %s", strerror(errno));
 902                         }
 903                         return;
 904                 }
 905                 rep.remote_addrlen = msg.msg_namelen;
 906                 sldns_buffer_skip(rep.c->buffer, rcv);
 907                 sldns_buffer_flip(rep.c->buffer);
 908                 rep.srctype = 0;
 909                 rep.is_proxied = 0;
 910 #ifndef S_SPLINT_S
 911                 for(cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
 912                         cmsg = CMSG_NXTHDR(&msg, cmsg)) {
 913                         if( cmsg->cmsg_level == IPPROTO_IPV6 &&
 914                                 cmsg->cmsg_type == IPV6_PKTINFO) {
 915                                 rep.srctype = 6;
 916                                 memmove(&rep.pktinfo.v6info, CMSG_DATA(cmsg),
 917                                         sizeof(struct in6_pktinfo));
 918                                 break;
 919 #ifdef IP_PKTINFO
 920                         } else if( cmsg->cmsg_level == IPPROTO_IP &&
 921                                 cmsg->cmsg_type == IP_PKTINFO) {
 922                                 rep.srctype = 4;
 923                                 memmove(&rep.pktinfo.v4info, CMSG_DATA(cmsg),
 924                                         sizeof(struct in_pktinfo));
 925                                 break;
 926 #elif defined(IP_RECVDSTADDR)
 927                         } else if( cmsg->cmsg_level == IPPROTO_IP &&
 928                                 cmsg->cmsg_type == IP_RECVDSTADDR) {
 929                                 rep.srctype = 4;
 930                                 memmove(&rep.pktinfo.v4addr, CMSG_DATA(cmsg),
 931                                         sizeof(struct in_addr));
 932                                 break;
 933 #endif /* IP_PKTINFO or IP_RECVDSTADDR */
 934 #ifdef HAVE_LINUX_NET_TSTAMP_H
 935                         } else if( cmsg->cmsg_level == SOL_SOCKET &&
 936                                 cmsg->cmsg_type == SO_TIMESTAMPNS) {
 937                                 ts = (struct timespec *)CMSG_DATA(cmsg);
 938                                 TIMESPEC_TO_TIMEVAL(&rep.c->recv_tv, ts);
 939                         } else if( cmsg->cmsg_level == SOL_SOCKET &&
 940                                 cmsg->cmsg_type == SO_TIMESTAMPING) {
 941                                 ts = (struct timespec *)CMSG_DATA(cmsg);
 942                                 TIMESPEC_TO_TIMEVAL(&rep.c->recv_tv, ts);
 943                         } else if( cmsg->cmsg_level == SOL_SOCKET &&
 944                                 cmsg->cmsg_type == SO_TIMESTAMP) {
 945                                 memmove(&rep.c->recv_tv, CMSG_DATA(cmsg), sizeof(struct timeval));
 946 #endif /* HAVE_LINUX_NET_TSTAMP_H */
 947                         }
 948                 }
 949
 950                 if(verbosity >= VERB_ALGO && rep.srctype != 0)
 951                         p_ancil("receive_udp on interface", &rep);
 952 #endif /* S_SPLINT_S */
 953
 954                 if(rep.c->pp2_enabled && !consume_pp2_header(rep.c->buffer,
 955                         &rep, 0)) {
 956                         log_err("proxy_protocol: could not consume PROXYv2 header");
 957                         return;
 958                 }
 959                 if(!rep.is_proxied) {
 960                         rep.client_addrlen = rep.remote_addrlen;
 961                         memmove(&rep.client_addr, &rep.remote_addr,
 962                                 rep.remote_addrlen);
 963                 }
 964
 965                 fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
 966                 if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
 967                         /* send back immediate reply */
 968                         (void)comm_point_send_udp_msg_if(rep.c, rep.c->buffer,
 969                                 (struct sockaddr*)&rep.remote_addr,
 970                                 rep.remote_addrlen, &rep);
 971                 }
 972                 if(!rep.c || rep.c->fd == -1) /* commpoint closed */
 973                         break;
 974         }
 975 #else
 976         (void)fd;
 977         (void)event;
 978         (void)arg;
 979         fatal_exit("recvmsg: No support for IPV6_PKTINFO; IP_PKTINFO or IP_RECVDSTADDR. "
 980                 "Please disable interface-automatic");
 981 #endif /* AF_INET6 && IPV6_PKTINFO && HAVE_RECVMSG */
 982 }
 983
 984 void
 985 comm_point_udp_callback(int fd, short event, void* arg)
 986 {
 987         struct comm_reply rep;
 988         ssize_t rcv;
 989         int i;
 990         struct sldns_buffer *buffer;
 991
 992         rep.c = (struct comm_point*)arg;
 993         log_assert(rep.c->type == comm_udp);
 994
 995         if(!(event&UB_EV_READ))
 996                 return;
 997         log_assert(rep.c && rep.c->buffer && rep.c->fd == fd);
 998         ub_comm_base_now(rep.c->ev->base);
 999         for(i=0; i<NUM_UDP_PER_SELECT; i++) {
1000                 sldns_buffer_clear(rep.c->buffer);
1001                 rep.remote_addrlen = (socklen_t)sizeof(rep.remote_addr);
1002                 log_assert(fd != -1);
1003                 log_assert(sldns_buffer_remaining(rep.c->buffer) > 0);
1004                 rcv = recvfrom(fd, (void*)sldns_buffer_begin(rep.c->buffer),
1005                         sldns_buffer_remaining(rep.c->buffer), MSG_DONTWAIT,
1006                         (struct sockaddr*)&rep.remote_addr, &rep.remote_addrlen);
1007                 if(rcv == -1) {
1008 #ifndef USE_WINSOCK
1009                         if(errno != EAGAIN && errno != EINTR
1010                                 && udp_recv_needs_log(errno))
1011                                 log_err("recvfrom %d failed: %s",
1012                                         fd, strerror(errno));
1013 #else
1014                         if(WSAGetLastError() != WSAEINPROGRESS &&
1015                                 WSAGetLastError() != WSAECONNRESET &&
1016                                 WSAGetLastError()!= WSAEWOULDBLOCK &&
1017                                 udp_recv_needs_log(WSAGetLastError()))
1018                                 log_err("recvfrom failed: %s",
1019                                         wsa_strerror(WSAGetLastError()));
1020 #endif
1021                         return;
1022                 }
1023                 sldns_buffer_skip(rep.c->buffer, rcv);
1024                 sldns_buffer_flip(rep.c->buffer);
1025                 rep.srctype = 0;
1026                 rep.is_proxied = 0;
1027
1028                 if(rep.c->pp2_enabled && !consume_pp2_header(rep.c->buffer,
1029                         &rep, 0)) {
1030                         log_err("proxy_protocol: could not consume PROXYv2 header");
1031                         return;
1032                 }
1033                 if(!rep.is_proxied) {
1034                         rep.client_addrlen = rep.remote_addrlen;
1035                         memmove(&rep.client_addr, &rep.remote_addr,
1036                                 rep.remote_addrlen);
1037                 }
1038
1039                 fptr_ok(fptr_whitelist_comm_point(rep.c->callback));
1040                 if((*rep.c->callback)(rep.c, rep.c->cb_arg, NETEVENT_NOERROR, &rep)) {
1041                         /* send back immediate reply */
1042 #ifdef USE_DNSCRYPT
1043                         buffer = rep.c->dnscrypt_buffer;
1044 #else
1045                         buffer = rep.c->buffer;
1046 #endif
1047                         (void)comm_point_send_udp_msg(rep.c, buffer,
1048                                 (struct sockaddr*)&rep.remote_addr,
1049                                 rep.remote_addrlen, 0);
1050                 }
1051                 if(!rep.c || rep.c->fd != fd) /* commpoint closed to -1 or reused for
1052                 another UDP port. Note rep.c cannot be reused with TCP fd. */
1053                         break;
1054         }
1055 }
1056
1057 int adjusted_tcp_timeout(struct comm_point* c)
1058 {
1059         if(c->tcp_timeout_msec < TCP_QUERY_TIMEOUT_MINIMUM)
1060                 return TCP_QUERY_TIMEOUT_MINIMUM;
1061         return c->tcp_timeout_msec;
1062 }
1063
1064 /** Use a new tcp handler for new query fd, set to read query */
1065 static void
1066 setup_tcp_handler(struct comm_point* c, int fd, int cur, int max)
1067 {
1068         int handler_usage;
1069         log_assert(c->type == comm_tcp || c->type == comm_http);
1070         log_assert(c->fd == -1);
1071         sldns_buffer_clear(c->buffer);
1072 #ifdef USE_DNSCRYPT
1073         if (c->dnscrypt)
1074                 sldns_buffer_clear(c->dnscrypt_buffer);
1075 #endif
1076         c->tcp_is_reading = 1;
1077         c->tcp_byte_count = 0;
1078         c->tcp_keepalive = 0;
1079         /* if more than half the tcp handlers are in use, use a shorter
1080          * timeout for this TCP connection, we need to make space for
1081          * other connections to be able to get attention */
1082         /* If > 50% TCP handler structures in use, set timeout to 1/100th
1083          *      configured value.
1084          * If > 65%TCP handler structures in use, set to 1/500th configured
1085          *      value.
1086          * If > 80% TCP handler structures in use, set to 0.
1087          *
1088          * If the timeout to use falls below 200 milliseconds, an actual
1089          * timeout of 200ms is used.
1090          */
1091         handler_usage = (cur * 100) / max;
1092         if(handler_usage > 50 && handler_usage <= 65)
1093                 c->tcp_timeout_msec /= 100;
1094         else if (handler_usage > 65 && handler_usage <= 80)
1095                 c->tcp_timeout_msec /= 500;
1096         else if (handler_usage > 80)
1097                 c->tcp_timeout_msec = 0;
1098         comm_point_start_listening(c, fd, adjusted_tcp_timeout(c));
1099 }
1100
1101 void comm_base_handle_slow_accept(int ATTR_UNUSED(fd),
1102         short ATTR_UNUSED(event), void* arg)
1103 {
1104         struct comm_base* b = (struct comm_base*)arg;
1105         /* timeout for the slow accept, re-enable accepts again */
1106         if(b->start_accept) {
1107                 verbose(VERB_ALGO, "wait is over, slow accept disabled");
1108                 fptr_ok(fptr_whitelist_start_accept(b->start_accept));
1109                 (*b->start_accept)(b->cb_arg);
1110                 b->eb->slow_accept_enabled = 0;
1111         }
1112 }
1113
1114 int comm_point_perform_accept(struct comm_point* c,
1115         struct sockaddr_storage* addr, socklen_t* addrlen)
1116 {
1117         int new_fd;
1118         *addrlen = (socklen_t)sizeof(*addr);
1119 #ifndef HAVE_ACCEPT4
1120         new_fd = accept(c->fd, (struct sockaddr*)addr, addrlen);
1121 #else
1122         /* SOCK_NONBLOCK saves extra calls to fcntl for the same result */
1123         new_fd = accept4(c->fd, (struct sockaddr*)addr, addrlen, SOCK_NONBLOCK);
1124 #endif
1125         if(new_fd == -1) {
1126 #ifndef USE_WINSOCK
1127                 /* EINTR is signal interrupt. others are closed connection. */
1128                 if(     errno == EINTR || errno == EAGAIN
1129 #ifdef EWOULDBLOCK
1130                         || errno == EWOULDBLOCK
1131 #endif
1132 #ifdef ECONNABORTED
1133                         || errno == ECONNABORTED
1134 #endif
1135 #ifdef EPROTO
1136                         || errno == EPROTO
1137 #endif /* EPROTO */
1138                         )
1139                         return -1;
1140 #if defined(ENFILE) && defined(EMFILE)
1141                 if(errno == ENFILE || errno == EMFILE) {
1142                         /* out of file descriptors, likely outside of our
1143                          * control. stop accept() calls for some time */
1144                         if(c->ev->base->stop_accept) {
1145                                 struct comm_base* b = c->ev->base;
1146                                 struct timeval tv;
1147                                 verbose(VERB_ALGO, "out of file descriptors: "
1148                                         "slow accept");
1149                                 ub_comm_base_now(b);
1150                                 if(b->eb->last_slow_log+SLOW_LOG_TIME <=
1151                                         b->eb->secs) {
1152                                         b->eb->last_slow_log = b->eb->secs;
1153                                         verbose(VERB_OPS, "accept failed, "
1154                                                 "slow down accept for %d "
1155                                                 "msec: %s",
1156                                                 NETEVENT_SLOW_ACCEPT_TIME,
1157                                                 sock_strerror(errno));
1158                                 }
1159                                 b->eb->slow_accept_enabled = 1;
1160                                 fptr_ok(fptr_whitelist_stop_accept(
1161                                         b->stop_accept));
1162                                 (*b->stop_accept)(b->cb_arg);
1163                                 /* set timeout, no mallocs */
1164                                 tv.tv_sec = NETEVENT_SLOW_ACCEPT_TIME/1000;
1165                                 tv.tv_usec = (NETEVENT_SLOW_ACCEPT_TIME%1000)*1000;
1166                                 b->eb->slow_accept = ub_event_new(b->eb->base,
1167                                         -1, UB_EV_TIMEOUT,
1168                                         comm_base_handle_slow_accept, b);
1169                                 if(b->eb->slow_accept == NULL) {
1170                                         /* we do not want to log here, because
1171                                          * that would spam the logfiles.
1172                                          * error: "event_base_set failed." */
1173                                 }
1174                                 else if(ub_event_add(b->eb->slow_accept, &tv)
1175                                         != 0) {
1176                                         /* we do not want to log here,
1177                                          * error: "event_add failed." */
1178                                 }
1179                         } else {
1180                                 log_err("accept, with no slow down, "
1181                                         "failed: %s", sock_strerror(errno));
1182                         }
1183                         return -1;
1184                 }
1185 #endif
1186 #else /* USE_WINSOCK */
1187                 if(WSAGetLastError() == WSAEINPROGRESS ||
1188                         WSAGetLastError() == WSAECONNRESET)
1189                         return -1;
1190                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
1191                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1192                         return -1;
1193                 }
1194 #endif
1195                 log_err_addr("accept failed", sock_strerror(errno), addr,
1196                         *addrlen);
1197                 return -1;
1198         }
1199         if(c->tcp_conn_limit && c->type == comm_tcp_accept) {
1200                 c->tcl_addr = tcl_addr_lookup(c->tcp_conn_limit, addr, *addrlen);
1201                 if(!tcl_new_connection(c->tcl_addr)) {
1202                         if(verbosity >= 3)
1203                                 log_err_addr("accept rejected",
1204                                 "connection limit exceeded", addr, *addrlen);
1205                         close(new_fd);
1206                         return -1;
1207                 }
1208         }
1209 #ifndef HAVE_ACCEPT4
1210         fd_set_nonblock(new_fd);
1211 #endif
1212         return new_fd;
1213 }
1214
1215 #ifdef USE_WINSOCK
1216 static long win_bio_cb(BIO *b, int oper, const char* ATTR_UNUSED(argp),
1217 #ifdef HAVE_BIO_SET_CALLBACK_EX
1218         size_t ATTR_UNUSED(len),
1219 #endif
1220         int ATTR_UNUSED(argi), long argl,
1221 #ifndef HAVE_BIO_SET_CALLBACK_EX
1222         long retvalue
1223 #else
1224         int retvalue, size_t* ATTR_UNUSED(processed)
1225 #endif
1226         )
1227 {
1228         int wsa_err = WSAGetLastError(); /* store errcode before it is gone */
1229         verbose(VERB_ALGO, "bio_cb %d, %s %s %s", oper,
1230                 (oper&BIO_CB_RETURN)?"return":"before",
1231                 (oper&BIO_CB_READ)?"read":((oper&BIO_CB_WRITE)?"write":"other"),
1232                 wsa_err==WSAEWOULDBLOCK?"wsawb":"");
1233         /* on windows, check if previous operation caused EWOULDBLOCK */
1234         if( (oper == (BIO_CB_READ|BIO_CB_RETURN) && argl == 0) ||
1235                 (oper == (BIO_CB_GETS|BIO_CB_RETURN) && argl == 0)) {
1236                 if(wsa_err == WSAEWOULDBLOCK)
1237                         ub_winsock_tcp_wouldblock((struct ub_event*)
1238                                 BIO_get_callback_arg(b), UB_EV_READ);
1239         }
1240         if( (oper == (BIO_CB_WRITE|BIO_CB_RETURN) && argl == 0) ||
1241                 (oper == (BIO_CB_PUTS|BIO_CB_RETURN) && argl == 0)) {
1242                 if(wsa_err == WSAEWOULDBLOCK)
1243                         ub_winsock_tcp_wouldblock((struct ub_event*)
1244                                 BIO_get_callback_arg(b), UB_EV_WRITE);
1245         }
1246         /* return original return value */
1247         return retvalue;
1248 }
1249
1250 /** set win bio callbacks for nonblocking operations */
1251 void
1252 comm_point_tcp_win_bio_cb(struct comm_point* c, void* thessl)
1253 {
1254         SSL* ssl = (SSL*)thessl;
1255         /* set them both just in case, but usually they are the same BIO */
1256 #ifdef HAVE_BIO_SET_CALLBACK_EX
1257         BIO_set_callback_ex(SSL_get_rbio(ssl), &win_bio_cb);
1258 #else
1259         BIO_set_callback(SSL_get_rbio(ssl), &win_bio_cb);
1260 #endif
1261         BIO_set_callback_arg(SSL_get_rbio(ssl), (char*)c->ev->ev);
1262 #ifdef HAVE_BIO_SET_CALLBACK_EX
1263         BIO_set_callback_ex(SSL_get_wbio(ssl), &win_bio_cb);
1264 #else
1265         BIO_set_callback(SSL_get_wbio(ssl), &win_bio_cb);
1266 #endif
1267         BIO_set_callback_arg(SSL_get_wbio(ssl), (char*)c->ev->ev);
1268 }
1269 #endif
1270
1271 #ifdef HAVE_NGHTTP2
1272 /** Create http2 session server.  Per connection, after TCP accepted.*/
1273 static int http2_session_server_create(struct http2_session* h2_session)
1274 {
1275         log_assert(h2_session->callbacks);
1276         h2_session->is_drop = 0;
1277         if(nghttp2_session_server_new(&h2_session->session,
1278                         h2_session->callbacks,
1279                 h2_session) == NGHTTP2_ERR_NOMEM) {
1280                 log_err("failed to create nghttp2 session server");
1281                 return 0;
1282         }
1283
1284         return 1;
1285 }
1286
1287 /** Submit http2 setting to session. Once per session. */
1288 static int http2_submit_settings(struct http2_session* h2_session)
1289 {
1290         int ret;
1291         nghttp2_settings_entry settings[1] = {
1292                 {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS,
1293                  h2_session->c->http2_max_streams}};
1294
1295         ret = nghttp2_submit_settings(h2_session->session, NGHTTP2_FLAG_NONE,
1296                 settings, 1);
1297         if(ret) {
1298                 verbose(VERB_QUERY, "http2: submit_settings failed, "
1299                         "error: %s", nghttp2_strerror(ret));
1300                 return 0;
1301         }
1302         return 1;
1303 }
1304 #endif /* HAVE_NGHTTP2 */
1305
1306
1307 void
1308 comm_point_tcp_accept_callback(int fd, short event, void* arg)
1309 {
1310         struct comm_point* c = (struct comm_point*)arg, *c_hdl;
1311         int new_fd;
1312         log_assert(c->type == comm_tcp_accept);
1313         if(!(event & UB_EV_READ)) {
1314                 log_info("ignoring tcp accept event %d", (int)event);
1315                 return;
1316         }
1317         ub_comm_base_now(c->ev->base);
1318         /* find free tcp handler. */
1319         if(!c->tcp_free) {
1320                 log_warn("accepted too many tcp, connections full");
1321                 return;
1322         }
1323         /* accept incoming connection. */
1324         c_hdl = c->tcp_free;
1325         /* clear leftover flags from previous use, and then set the
1326          * correct event base for the event structure for libevent */
1327         ub_event_free(c_hdl->ev->ev);
1328         c_hdl->ev->ev = NULL;
1329         if((c_hdl->type == comm_tcp && c_hdl->tcp_req_info) ||
1330                 c_hdl->type == comm_local || c_hdl->type == comm_raw)
1331                 c_hdl->tcp_do_toggle_rw = 0;
1332         else    c_hdl->tcp_do_toggle_rw = 1;
1333
1334         if(c_hdl->type == comm_http) {
1335 #ifdef HAVE_NGHTTP2
1336                 if(!c_hdl->h2_session ||
1337                         !http2_session_server_create(c_hdl->h2_session)) {
1338                         log_warn("failed to create nghttp2");
1339                         return;
1340                 }
1341                 if(!c_hdl->h2_session ||
1342                         !http2_submit_settings(c_hdl->h2_session)) {
1343                         log_warn("failed to submit http2 settings");
1344                         return;
1345                 }
1346                 if(!c->ssl) {
1347                         c_hdl->tcp_do_toggle_rw = 0;
1348                         c_hdl->use_h2 = 1;
1349                 }
1350 #endif
1351                 c_hdl->ev->ev = ub_event_new(c_hdl->ev->base->eb->base, -1,
1352                         UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT,
1353                         comm_point_http_handle_callback, c_hdl);
1354         } else {
1355                 c_hdl->ev->ev = ub_event_new(c_hdl->ev->base->eb->base, -1,
1356                         UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT,
1357                         comm_point_tcp_handle_callback, c_hdl);
1358         }
1359         if(!c_hdl->ev->ev) {
1360                 log_warn("could not ub_event_new, dropped tcp");
1361                 return;
1362         }
1363         log_assert(fd != -1);
1364         (void)fd;
1365         new_fd = comm_point_perform_accept(c, &c_hdl->repinfo.remote_addr,
1366                 &c_hdl->repinfo.remote_addrlen);
1367         if(new_fd == -1)
1368                 return;
1369         /* Copy remote_address to client_address.
1370          * Simplest way/time for streams to do that. */
1371         c_hdl->repinfo.client_addrlen = c_hdl->repinfo.remote_addrlen;
1372         memmove(&c_hdl->repinfo.client_addr,
1373                 &c_hdl->repinfo.remote_addr,
1374                 c_hdl->repinfo.remote_addrlen);
1375         if(c->ssl) {
1376                 c_hdl->ssl = incoming_ssl_fd(c->ssl, new_fd);
1377                 if(!c_hdl->ssl) {
1378                         c_hdl->fd = new_fd;
1379                         comm_point_close(c_hdl);
1380                         return;
1381                 }
1382                 c_hdl->ssl_shake_state = comm_ssl_shake_read;
1383 #ifdef USE_WINSOCK
1384                 comm_point_tcp_win_bio_cb(c_hdl, c_hdl->ssl);
1385 #endif
1386         }
1387
1388         /* grab the tcp handler buffers */
1389         c->cur_tcp_count++;
1390         c->tcp_free = c_hdl->tcp_free;
1391         c_hdl->tcp_free = NULL;
1392         if(!c->tcp_free) {
1393                 /* stop accepting incoming queries for now. */
1394                 comm_point_stop_listening(c);
1395         }
1396         setup_tcp_handler(c_hdl, new_fd, c->cur_tcp_count, c->max_tcp_count);
1397 }
1398
1399 /** Make tcp handler free for next assignment */
1400 static void
1401 reclaim_tcp_handler(struct comm_point* c)
1402 {
1403         log_assert(c->type == comm_tcp);
1404         if(c->ssl) {
1405 #ifdef HAVE_SSL
1406                 SSL_shutdown(c->ssl);
1407                 SSL_free(c->ssl);
1408                 c->ssl = NULL;
1409 #endif
1410         }
1411         comm_point_close(c);
1412         if(c->tcp_parent) {
1413                 if(c != c->tcp_parent->tcp_free) {
1414                         c->tcp_parent->cur_tcp_count--;
1415                         c->tcp_free = c->tcp_parent->tcp_free;
1416                         c->tcp_parent->tcp_free = c;
1417                 }
1418                 if(!c->tcp_free) {
1419                         /* re-enable listening on accept socket */
1420                         comm_point_start_listening(c->tcp_parent, -1, -1);
1421                 }
1422         }
1423         c->tcp_more_read_again = NULL;
1424         c->tcp_more_write_again = NULL;
1425         c->tcp_byte_count = 0;
1426         c->pp2_header_state = pp2_header_none;
1427         sldns_buffer_clear(c->buffer);
1428 }
1429
1430 /** do the callback when writing is done */
1431 static void
1432 tcp_callback_writer(struct comm_point* c)
1433 {
1434         log_assert(c->type == comm_tcp);
1435         if(!c->tcp_write_and_read) {
1436                 sldns_buffer_clear(c->buffer);
1437                 c->tcp_byte_count = 0;
1438         }
1439         if(c->tcp_do_toggle_rw)
1440                 c->tcp_is_reading = 1;
1441         /* switch from listening(write) to listening(read) */
1442         if(c->tcp_req_info) {
1443                 tcp_req_info_handle_writedone(c->tcp_req_info);
1444         } else {
1445                 comm_point_stop_listening(c);
1446                 if(c->tcp_write_and_read) {
1447                         fptr_ok(fptr_whitelist_comm_point(c->callback));
1448                         if( (*c->callback)(c, c->cb_arg, NETEVENT_PKT_WRITTEN,
1449                                 &c->repinfo) ) {
1450                                 comm_point_start_listening(c, -1,
1451                                         adjusted_tcp_timeout(c));
1452                         }
1453                 } else {
1454                         comm_point_start_listening(c, -1,
1455                                         adjusted_tcp_timeout(c));
1456                 }
1457         }
1458 }
1459
1460 /** do the callback when reading is done */
1461 static void
1462 tcp_callback_reader(struct comm_point* c)
1463 {
1464         log_assert(c->type == comm_tcp || c->type == comm_local);
1465         sldns_buffer_flip(c->buffer);
1466         if(c->tcp_do_toggle_rw)
1467                 c->tcp_is_reading = 0;
1468         c->tcp_byte_count = 0;
1469         if(c->tcp_req_info) {
1470                 tcp_req_info_handle_readdone(c->tcp_req_info);
1471         } else {
1472                 if(c->type == comm_tcp)
1473                         comm_point_stop_listening(c);
1474                 fptr_ok(fptr_whitelist_comm_point(c->callback));
1475                 if( (*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &c->repinfo) ) {
1476                         comm_point_start_listening(c, -1,
1477                                         adjusted_tcp_timeout(c));
1478                 }
1479         }
1480 }
1481
1482 #ifdef HAVE_SSL
1483 /** true if the ssl handshake error has to be squelched from the logs */
1484 int
1485 squelch_err_ssl_handshake(unsigned long err)
1486 {
1487         if(verbosity >= VERB_QUERY)
1488                 return 0; /* only squelch on low verbosity */
1489         if(ERR_GET_LIB(err) == ERR_LIB_SSL &&
1490                 (ERR_GET_REASON(err) == SSL_R_HTTPS_PROXY_REQUEST ||
1491                  ERR_GET_REASON(err) == SSL_R_HTTP_REQUEST ||
1492                  ERR_GET_REASON(err) == SSL_R_WRONG_VERSION_NUMBER ||
1493                  ERR_GET_REASON(err) == SSL_R_SSLV3_ALERT_BAD_CERTIFICATE
1494 #ifdef SSL_F_TLS_POST_PROCESS_CLIENT_HELLO
1495                  || ERR_GET_REASON(err) == SSL_R_NO_SHARED_CIPHER
1496 #endif
1497 #ifdef SSL_F_TLS_EARLY_POST_PROCESS_CLIENT_HELLO
1498                  || ERR_GET_REASON(err) == SSL_R_UNKNOWN_PROTOCOL
1499                  || ERR_GET_REASON(err) == SSL_R_UNSUPPORTED_PROTOCOL
1500 #  ifdef SSL_R_VERSION_TOO_LOW
1501                  || ERR_GET_REASON(err) == SSL_R_VERSION_TOO_LOW
1502 #  endif
1503 #endif
1504                 ))
1505                 return 1;
1506         return 0;
1507 }
1508 #endif /* HAVE_SSL */
1509
1510 /** continue ssl handshake */
1511 #ifdef HAVE_SSL
1512 static int
1513 ssl_handshake(struct comm_point* c)
1514 {
1515         int r;
1516         if(c->ssl_shake_state == comm_ssl_shake_hs_read) {
1517                 /* read condition satisfied back to writing */
1518                 comm_point_listen_for_rw(c, 0, 1);
1519                 c->ssl_shake_state = comm_ssl_shake_none;
1520                 return 1;
1521         }
1522         if(c->ssl_shake_state == comm_ssl_shake_hs_write) {
1523                 /* write condition satisfied, back to reading */
1524                 comm_point_listen_for_rw(c, 1, 0);
1525                 c->ssl_shake_state = comm_ssl_shake_none;
1526                 return 1;
1527         }
1528
1529         ERR_clear_error();
1530         r = SSL_do_handshake(c->ssl);
1531         if(r != 1) {
1532                 int want = SSL_get_error(c->ssl, r);
1533                 if(want == SSL_ERROR_WANT_READ) {
1534                         if(c->ssl_shake_state == comm_ssl_shake_read)
1535                                 return 1;
1536                         c->ssl_shake_state = comm_ssl_shake_read;
1537                         comm_point_listen_for_rw(c, 1, 0);
1538                         return 1;
1539                 } else if(want == SSL_ERROR_WANT_WRITE) {
1540                         if(c->ssl_shake_state == comm_ssl_shake_write)
1541                                 return 1;
1542                         c->ssl_shake_state = comm_ssl_shake_write;
1543                         comm_point_listen_for_rw(c, 0, 1);
1544                         return 1;
1545                 } else if(r == 0) {
1546                         return 0; /* closed */
1547                 } else if(want == SSL_ERROR_SYSCALL) {
1548                         /* SYSCALL and errno==0 means closed uncleanly */
1549 #ifdef EPIPE
1550                         if(errno == EPIPE && verbosity < 2)
1551                                 return 0; /* silence 'broken pipe' */
1552 #endif
1553 #ifdef ECONNRESET
1554                         if(errno == ECONNRESET && verbosity < 2)
1555                                 return 0; /* silence reset by peer */
1556 #endif
1557                         if(!tcp_connect_errno_needs_log(
1558                                 (struct sockaddr*)&c->repinfo.remote_addr,
1559                                 c->repinfo.remote_addrlen))
1560                                 return 0; /* silence connect failures that
1561                                 show up because after connect this is the
1562                                 first system call that accesses the socket */
1563                         if(errno != 0)
1564                                 log_err("SSL_handshake syscall: %s",
1565                                         strerror(errno));
1566                         return 0;
1567                 } else {
1568                         unsigned long err = ERR_get_error();
1569                         if(!squelch_err_ssl_handshake(err)) {
1570                                 log_crypto_err_code("ssl handshake failed", err);
1571                                 log_addr(VERB_OPS, "ssl handshake failed",
1572                                         &c->repinfo.remote_addr,
1573                                         c->repinfo.remote_addrlen);
1574                         }
1575                         return 0;
1576                 }
1577         }
1578         /* this is where peer verification could take place */
1579         if((SSL_get_verify_mode(c->ssl)&SSL_VERIFY_PEER)) {
1580                 /* verification */
1581                 if(SSL_get_verify_result(c->ssl) == X509_V_OK) {
1582 #ifdef HAVE_SSL_GET1_PEER_CERTIFICATE
1583                         X509* x = SSL_get1_peer_certificate(c->ssl);
1584 #else
1585                         X509* x = SSL_get_peer_certificate(c->ssl);
1586 #endif
1587                         if(!x) {
1588                                 log_addr(VERB_ALGO, "SSL connection failed: "
1589                                         "no certificate",
1590                                         &c->repinfo.remote_addr,
1591                                         c->repinfo.remote_addrlen);
1592                                 return 0;
1593                         }
1594                         log_cert(VERB_ALGO, "peer certificate", x);
1595 #ifdef HAVE_SSL_GET0_PEERNAME
1596                         if(SSL_get0_peername(c->ssl)) {
1597                                 char buf[255];
1598                                 snprintf(buf, sizeof(buf), "SSL connection "
1599                                         "to %s authenticated",
1600                                         SSL_get0_peername(c->ssl));
1601                                 log_addr(VERB_ALGO, buf, &c->repinfo.remote_addr,
1602                                         c->repinfo.remote_addrlen);
1603                         } else {
1604 #endif
1605                                 log_addr(VERB_ALGO, "SSL connection "
1606                                         "authenticated", &c->repinfo.remote_addr,
1607                                         c->repinfo.remote_addrlen);
1608 #ifdef HAVE_SSL_GET0_PEERNAME
1609                         }
1610 #endif
1611                         X509_free(x);
1612                 } else {
1613 #ifdef HAVE_SSL_GET1_PEER_CERTIFICATE
1614                         X509* x = SSL_get1_peer_certificate(c->ssl);
1615 #else
1616                         X509* x = SSL_get_peer_certificate(c->ssl);
1617 #endif
1618                         if(x) {
1619                                 log_cert(VERB_ALGO, "peer certificate", x);
1620                                 X509_free(x);
1621                         }
1622                         log_addr(VERB_ALGO, "SSL connection failed: "
1623                                 "failed to authenticate",
1624                                 &c->repinfo.remote_addr,
1625                                 c->repinfo.remote_addrlen);
1626                         return 0;
1627                 }
1628         } else {
1629                 /* unauthenticated, the verify peer flag was not set
1630                  * in c->ssl when the ssl object was created from ssl_ctx */
1631                 log_addr(VERB_ALGO, "SSL connection", &c->repinfo.remote_addr,
1632                         c->repinfo.remote_addrlen);
1633         }
1634
1635 #ifdef HAVE_SSL_GET0_ALPN_SELECTED
1636         /* check if http2 use is negotiated */
1637         if(c->type == comm_http && c->h2_session) {
1638                 const unsigned char *alpn;
1639                 unsigned int alpnlen = 0;
1640                 SSL_get0_alpn_selected(c->ssl, &alpn, &alpnlen);
1641                 if(alpnlen == 2 && memcmp("h2", alpn, 2) == 0) {
1642                         /* connection upgraded to HTTP2 */
1643                         c->tcp_do_toggle_rw = 0;
1644                         c->use_h2 = 1;
1645                 }
1646         }
1647 #endif
1648
1649         /* setup listen rw correctly */
1650         if(c->tcp_is_reading) {
1651                 if(c->ssl_shake_state != comm_ssl_shake_read)
1652                         comm_point_listen_for_rw(c, 1, 0);
1653         } else {
1654                 comm_point_listen_for_rw(c, 0, 1);
1655         }
1656         c->ssl_shake_state = comm_ssl_shake_none;
1657         return 1;
1658 }
1659 #endif /* HAVE_SSL */
1660
1661 /** ssl read callback on TCP */
1662 static int
1663 ssl_handle_read(struct comm_point* c)
1664 {
1665 #ifdef HAVE_SSL
1666         int r;
1667         if(c->ssl_shake_state != comm_ssl_shake_none) {
1668                 if(!ssl_handshake(c))
1669                         return 0;
1670                 if(c->ssl_shake_state != comm_ssl_shake_none)
1671                         return 1;
1672         }
1673         if(c->pp2_enabled && c->pp2_header_state != pp2_header_done) {
1674                 struct pp2_header* header = NULL;
1675                 size_t want_read_size = 0;
1676                 size_t current_read_size = 0;
1677                 if(c->pp2_header_state == pp2_header_none) {
1678                         want_read_size = PP2_HEADER_SIZE;
1679                         if(sldns_buffer_remaining(c->buffer)<want_read_size) {
1680                                 log_err_addr("proxy_protocol: not enough "
1681                                         "buffer size to read PROXYv2 header", "",
1682                                         &c->repinfo.remote_addr,
1683                                         c->repinfo.remote_addrlen);
1684                                 return 0;
1685                         }
1686                         verbose(VERB_ALGO, "proxy_protocol: reading fixed "
1687                                 "part of PROXYv2 header (len %lu)",
1688                                 (unsigned long)want_read_size);
1689                         current_read_size = want_read_size;
1690                         if(c->tcp_byte_count < current_read_size) {
1691                                 ERR_clear_error();
1692                                 if((r=SSL_read(c->ssl, (void*)sldns_buffer_at(
1693                                         c->buffer, c->tcp_byte_count),
1694                                         current_read_size -
1695                                         c->tcp_byte_count)) <= 0) {
1696                                         int want = SSL_get_error(c->ssl, r);
1697                                         if(want == SSL_ERROR_ZERO_RETURN) {
1698                                                 if(c->tcp_req_info)
1699                                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1700                                                 return 0; /* shutdown, closed */
1701                                         } else if(want == SSL_ERROR_WANT_READ) {
1702 #ifdef USE_WINSOCK
1703                                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1704 #endif
1705                                                 return 1; /* read more later */
1706                                         } else if(want == SSL_ERROR_WANT_WRITE) {
1707                                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1708                                                 comm_point_listen_for_rw(c, 0, 1);
1709                                                 return 1;
1710                                         } else if(want == SSL_ERROR_SYSCALL) {
1711 #ifdef ECONNRESET
1712                                                 if(errno == ECONNRESET && verbosity < 2)
1713                                                         return 0; /* silence reset by peer */
1714 #endif
1715                                                 if(errno != 0)
1716                                                         log_err("SSL_read syscall: %s",
1717                                                                 strerror(errno));
1718                                                 return 0;
1719                                         }
1720                                         log_crypto_err("could not SSL_read");
1721                                         return 0;
1722                                 }
1723                                 c->tcp_byte_count += r;
1724                                 if(c->tcp_byte_count != current_read_size) return 1;
1725                                 c->pp2_header_state = pp2_header_init;
1726                         }
1727                 }
1728                 if(c->pp2_header_state == pp2_header_init) {
1729                         header = pp2_read_header(c->buffer);
1730                         if(!header) {
1731                                 log_err("proxy_protocol: could not parse "
1732                                         "PROXYv2 header");
1733                                 return 0;
1734                         }
1735                         want_read_size = ntohs(header->len);
1736                         if(sldns_buffer_remaining(c->buffer) <
1737                                 PP2_HEADER_SIZE + want_read_size) {
1738                                 log_err_addr("proxy_protocol: not enough "
1739                                         "buffer size to read PROXYv2 header", "",
1740                                         &c->repinfo.remote_addr,
1741                                         c->repinfo.remote_addrlen);
1742                                 return 0;
1743                         }
1744                         verbose(VERB_ALGO, "proxy_protocol: reading variable "
1745                                 "part of PROXYv2 header (len %lu)",
1746                                 (unsigned long)want_read_size);
1747                         current_read_size = PP2_HEADER_SIZE + want_read_size;
1748                         if(want_read_size == 0) {
1749                                 /* nothing more to read; header is complete */
1750                                 c->pp2_header_state = pp2_header_done;
1751                         } else if(c->tcp_byte_count < current_read_size) {
1752                                 ERR_clear_error();
1753                                 if((r=SSL_read(c->ssl, (void*)sldns_buffer_at(
1754                                         c->buffer, c->tcp_byte_count),
1755                                         current_read_size -
1756                                         c->tcp_byte_count)) <= 0) {
1757                                         int want = SSL_get_error(c->ssl, r);
1758                                         if(want == SSL_ERROR_ZERO_RETURN) {
1759                                                 if(c->tcp_req_info)
1760                                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1761                                                 return 0; /* shutdown, closed */
1762                                         } else if(want == SSL_ERROR_WANT_READ) {
1763 #ifdef USE_WINSOCK
1764                                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1765 #endif
1766                                                 return 1; /* read more later */
1767                                         } else if(want == SSL_ERROR_WANT_WRITE) {
1768                                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1769                                                 comm_point_listen_for_rw(c, 0, 1);
1770                                                 return 1;
1771                                         } else if(want == SSL_ERROR_SYSCALL) {
1772 #ifdef ECONNRESET
1773                                                 if(errno == ECONNRESET && verbosity < 2)
1774                                                         return 0; /* silence reset by peer */
1775 #endif
1776                                                 if(errno != 0)
1777                                                         log_err("SSL_read syscall: %s",
1778                                                                 strerror(errno));
1779                                                 return 0;
1780                                         }
1781                                         log_crypto_err("could not SSL_read");
1782                                         return 0;
1783                                 }
1784                                 c->tcp_byte_count += r;
1785                                 if(c->tcp_byte_count != current_read_size) return 1;
1786                                 c->pp2_header_state = pp2_header_done;
1787                         }
1788                 }
1789                 if(c->pp2_header_state != pp2_header_done || !header) {
1790                         log_err_addr("proxy_protocol: wrong state for the "
1791                                 "PROXYv2 header", "", &c->repinfo.remote_addr,
1792                                 c->repinfo.remote_addrlen);
1793                         return 0;
1794                 }
1795                 if(!consume_pp2_header(c->buffer, &c->repinfo, 1)) {
1796                         log_err_addr("proxy_protocol: could not consume "
1797                                 "PROXYv2 header", "", &c->repinfo.remote_addr,
1798                                 c->repinfo.remote_addrlen);
1799                         return 0;
1800                 }
1801                 verbose(VERB_ALGO, "proxy_protocol: successful read of "
1802                         "PROXYv2 header");
1803                 /* Clear and reset the buffer to read the following
1804                  * DNS packet(s). */
1805                 sldns_buffer_clear(c->buffer);
1806                 c->tcp_byte_count = 0;
1807                 return 1;
1808         }
1809         if(c->tcp_byte_count < sizeof(uint16_t)) {
1810                 /* read length bytes */
1811                 ERR_clear_error();
1812                 if((r=SSL_read(c->ssl, (void*)sldns_buffer_at(c->buffer,
1813                         c->tcp_byte_count), (int)(sizeof(uint16_t) -
1814                         c->tcp_byte_count))) <= 0) {
1815                         int want = SSL_get_error(c->ssl, r);
1816                         if(want == SSL_ERROR_ZERO_RETURN) {
1817                                 if(c->tcp_req_info)
1818                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1819                                 return 0; /* shutdown, closed */
1820                         } else if(want == SSL_ERROR_WANT_READ) {
1821 #ifdef USE_WINSOCK
1822                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1823 #endif
1824                                 return 1; /* read more later */
1825                         } else if(want == SSL_ERROR_WANT_WRITE) {
1826                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1827                                 comm_point_listen_for_rw(c, 0, 1);
1828                                 return 1;
1829                         } else if(want == SSL_ERROR_SYSCALL) {
1830 #ifdef ECONNRESET
1831                                 if(errno == ECONNRESET && verbosity < 2)
1832                                         return 0; /* silence reset by peer */
1833 #endif
1834                                 if(errno != 0)
1835                                         log_err("SSL_read syscall: %s",
1836                                                 strerror(errno));
1837                                 return 0;
1838                         }
1839                         log_crypto_err("could not SSL_read");
1840                         return 0;
1841                 }
1842                 c->tcp_byte_count += r;
1843                 if(c->tcp_byte_count < sizeof(uint16_t))
1844                         return 1;
1845                 if(sldns_buffer_read_u16_at(c->buffer, 0) >
1846                         sldns_buffer_capacity(c->buffer)) {
1847                         verbose(VERB_QUERY, "ssl: dropped larger than buffer");
1848                         return 0;
1849                 }
1850                 sldns_buffer_set_limit(c->buffer,
1851                         sldns_buffer_read_u16_at(c->buffer, 0));
1852                 if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1853                         verbose(VERB_QUERY, "ssl: dropped bogus too short.");
1854                         return 0;
1855                 }
1856                 sldns_buffer_skip(c->buffer, (ssize_t)(c->tcp_byte_count-sizeof(uint16_t)));
1857                 verbose(VERB_ALGO, "Reading ssl tcp query of length %d",
1858                         (int)sldns_buffer_limit(c->buffer));
1859         }
1860         if(sldns_buffer_remaining(c->buffer) > 0) {
1861                 ERR_clear_error();
1862                 r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
1863                         (int)sldns_buffer_remaining(c->buffer));
1864                 if(r <= 0) {
1865                         int want = SSL_get_error(c->ssl, r);
1866                         if(want == SSL_ERROR_ZERO_RETURN) {
1867                                 if(c->tcp_req_info)
1868                                         return tcp_req_info_handle_read_close(c->tcp_req_info);
1869                                 return 0; /* shutdown, closed */
1870                         } else if(want == SSL_ERROR_WANT_READ) {
1871 #ifdef USE_WINSOCK
1872                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
1873 #endif
1874                                 return 1; /* read more later */
1875                         } else if(want == SSL_ERROR_WANT_WRITE) {
1876                                 c->ssl_shake_state = comm_ssl_shake_hs_write;
1877                                 comm_point_listen_for_rw(c, 0, 1);
1878                                 return 1;
1879                         } else if(want == SSL_ERROR_SYSCALL) {
1880 #ifdef ECONNRESET
1881                                 if(errno == ECONNRESET && verbosity < 2)
1882                                         return 0; /* silence reset by peer */
1883 #endif
1884                                 if(errno != 0)
1885                                         log_err("SSL_read syscall: %s",
1886                                                 strerror(errno));
1887                                 return 0;
1888                         }
1889                         log_crypto_err("could not SSL_read");
1890                         return 0;
1891                 }
1892                 sldns_buffer_skip(c->buffer, (ssize_t)r);
1893         }
1894         if(sldns_buffer_remaining(c->buffer) <= 0) {
1895                 tcp_callback_reader(c);
1896         }
1897         return 1;
1898 #else
1899         (void)c;
1900         return 0;
1901 #endif /* HAVE_SSL */
1902 }
1903
1904 /** ssl write callback on TCP */
1905 static int
1906 ssl_handle_write(struct comm_point* c)
1907 {
1908 #ifdef HAVE_SSL
1909         int r;
1910         if(c->ssl_shake_state != comm_ssl_shake_none) {
1911                 if(!ssl_handshake(c))
1912                         return 0;
1913                 if(c->ssl_shake_state != comm_ssl_shake_none)
1914                         return 1;
1915         }
1916         /* ignore return, if fails we may simply block */
1917         (void)SSL_set_mode(c->ssl, (long)SSL_MODE_ENABLE_PARTIAL_WRITE);
1918         if((c->tcp_write_and_read?c->tcp_write_byte_count:c->tcp_byte_count) < sizeof(uint16_t)) {
1919                 uint16_t len = htons(c->tcp_write_and_read?c->tcp_write_pkt_len:sldns_buffer_limit(c->buffer));
1920                 ERR_clear_error();
1921                 if(c->tcp_write_and_read) {
1922                         if(c->tcp_write_pkt_len + 2 < LDNS_RR_BUF_SIZE) {
1923                                 /* combine the tcp length and the query for
1924                                  * write, this emulates writev */
1925                                 uint8_t buf[LDNS_RR_BUF_SIZE];
1926                                 memmove(buf, &len, sizeof(uint16_t));
1927                                 memmove(buf+sizeof(uint16_t),
1928                                         c->tcp_write_pkt,
1929                                         c->tcp_write_pkt_len);
1930                                 r = SSL_write(c->ssl,
1931                                         (void*)(buf+c->tcp_write_byte_count),
1932                                         c->tcp_write_pkt_len + 2 -
1933                                         c->tcp_write_byte_count);
1934                         } else {
1935                                 r = SSL_write(c->ssl,
1936                                         (void*)(((uint8_t*)&len)+c->tcp_write_byte_count),
1937                                         (int)(sizeof(uint16_t)-c->tcp_write_byte_count));
1938                         }
1939                 } else if(sizeof(uint16_t)+sldns_buffer_remaining(c->buffer) <
1940                         LDNS_RR_BUF_SIZE) {
1941                         /* combine the tcp length and the query for write,
1942                          * this emulates writev */
1943                         uint8_t buf[LDNS_RR_BUF_SIZE];
1944                         memmove(buf, &len, sizeof(uint16_t));
1945                         memmove(buf+sizeof(uint16_t),
1946                                 sldns_buffer_current(c->buffer),
1947                                 sldns_buffer_remaining(c->buffer));
1948                         r = SSL_write(c->ssl, (void*)(buf+c->tcp_byte_count),
1949                                 (int)(sizeof(uint16_t)+
1950                                 sldns_buffer_remaining(c->buffer)
1951                                 - c->tcp_byte_count));
1952                 } else {
1953                         r = SSL_write(c->ssl,
1954                                 (void*)(((uint8_t*)&len)+c->tcp_byte_count),
1955                                 (int)(sizeof(uint16_t)-c->tcp_byte_count));
1956                 }
1957                 if(r <= 0) {
1958                         int want = SSL_get_error(c->ssl, r);
1959                         if(want == SSL_ERROR_ZERO_RETURN) {
1960                                 return 0; /* closed */
1961                         } else if(want == SSL_ERROR_WANT_READ) {
1962                                 c->ssl_shake_state = comm_ssl_shake_hs_read;
1963                                 comm_point_listen_for_rw(c, 1, 0);
1964                                 return 1; /* wait for read condition */
1965                         } else if(want == SSL_ERROR_WANT_WRITE) {
1966 #ifdef USE_WINSOCK
1967                                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
1968 #endif
1969                                 return 1; /* write more later */
1970                         } else if(want == SSL_ERROR_SYSCALL) {
1971 #ifdef EPIPE
1972                                 if(errno == EPIPE && verbosity < 2)
1973                                         return 0; /* silence 'broken pipe' */
1974 #endif
1975                                 if(errno != 0)
1976                                         log_err("SSL_write syscall: %s",
1977                                                 strerror(errno));
1978                                 return 0;
1979                         }
1980                         log_crypto_err("could not SSL_write");
1981                         return 0;
1982                 }
1983                 if(c->tcp_write_and_read) {
1984                         c->tcp_write_byte_count += r;
1985                         if(c->tcp_write_byte_count < sizeof(uint16_t))
1986                                 return 1;
1987                 } else {
1988                         c->tcp_byte_count += r;
1989                         if(c->tcp_byte_count < sizeof(uint16_t))
1990                                 return 1;
1991                         sldns_buffer_set_position(c->buffer, c->tcp_byte_count -
1992                                 sizeof(uint16_t));
1993                 }
1994                 if((!c->tcp_write_and_read && sldns_buffer_remaining(c->buffer) == 0) || (c->tcp_write_and_read && c->tcp_write_byte_count == c->tcp_write_pkt_len + 2)) {
1995                         tcp_callback_writer(c);
1996                         return 1;
1997                 }
1998         }
1999         log_assert(c->tcp_write_and_read || sldns_buffer_remaining(c->buffer) > 0);
2000         log_assert(!c->tcp_write_and_read || c->tcp_write_byte_count < c->tcp_write_pkt_len + 2);
2001         ERR_clear_error();
2002         if(c->tcp_write_and_read) {
2003                 r = SSL_write(c->ssl, (void*)(c->tcp_write_pkt + c->tcp_write_byte_count - 2),
2004                         (int)(c->tcp_write_pkt_len + 2 - c->tcp_write_byte_count));
2005         } else {
2006                 r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
2007                         (int)sldns_buffer_remaining(c->buffer));
2008         }
2009         if(r <= 0) {
2010                 int want = SSL_get_error(c->ssl, r);
2011                 if(want == SSL_ERROR_ZERO_RETURN) {
2012                         return 0; /* closed */
2013                 } else if(want == SSL_ERROR_WANT_READ) {
2014                         c->ssl_shake_state = comm_ssl_shake_hs_read;
2015                         comm_point_listen_for_rw(c, 1, 0);
2016                         return 1; /* wait for read condition */
2017                 } else if(want == SSL_ERROR_WANT_WRITE) {
2018 #ifdef USE_WINSOCK
2019                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2020 #endif
2021                         return 1; /* write more later */
2022                 } else if(want == SSL_ERROR_SYSCALL) {
2023 #ifdef EPIPE
2024                         if(errno == EPIPE && verbosity < 2)
2025                                 return 0; /* silence 'broken pipe' */
2026 #endif
2027                         if(errno != 0)
2028                                 log_err("SSL_write syscall: %s",
2029                                         strerror(errno));
2030                         return 0;
2031                 }
2032                 log_crypto_err("could not SSL_write");
2033                 return 0;
2034         }
2035         if(c->tcp_write_and_read) {
2036                 c->tcp_write_byte_count += r;
2037         } else {
2038                 sldns_buffer_skip(c->buffer, (ssize_t)r);
2039         }
2040
2041         if((!c->tcp_write_and_read && sldns_buffer_remaining(c->buffer) == 0) || (c->tcp_write_and_read && c->tcp_write_byte_count == c->tcp_write_pkt_len + 2)) {
2042                 tcp_callback_writer(c);
2043         }
2044         return 1;
2045 #else
2046         (void)c;
2047         return 0;
2048 #endif /* HAVE_SSL */
2049 }
2050
2051 /** handle ssl tcp connection with dns contents */
2052 static int
2053 ssl_handle_it(struct comm_point* c, int is_write)
2054 {
2055         /* handle case where renegotiation wants read during write call
2056          * or write during read calls */
2057         if(is_write && c->ssl_shake_state == comm_ssl_shake_hs_write)
2058                 return ssl_handle_read(c);
2059         else if(!is_write && c->ssl_shake_state == comm_ssl_shake_hs_read)
2060                 return ssl_handle_write(c);
2061         /* handle read events for read operation and write events for a
2062          * write operation */
2063         else if(!is_write)
2064                 return ssl_handle_read(c);
2065         return ssl_handle_write(c);
2066 }
2067
2068 /**
2069  * Handle tcp reading callback.
2070  * @param fd: file descriptor of socket.
2071  * @param c: comm point to read from into buffer.
2072  * @param short_ok: if true, very short packets are OK (for comm_local).
2073  * @return: 0 on error
2074  */
2075 static int
2076 comm_point_tcp_handle_read(int fd, struct comm_point* c, int short_ok)
2077 {
2078         ssize_t r;
2079         int recv_initial = 0;
2080         log_assert(c->type == comm_tcp || c->type == comm_local);
2081         if(c->ssl)
2082                 return ssl_handle_it(c, 0);
2083         if(!c->tcp_is_reading && !c->tcp_write_and_read)
2084                 return 0;
2085
2086         log_assert(fd != -1);
2087         if(c->pp2_enabled && c->pp2_header_state != pp2_header_done) {
2088                 struct pp2_header* header = NULL;
2089                 size_t want_read_size = 0;
2090                 size_t current_read_size = 0;
2091                 if(c->pp2_header_state == pp2_header_none) {
2092                         want_read_size = PP2_HEADER_SIZE;
2093                         if(sldns_buffer_remaining(c->buffer)<want_read_size) {
2094                                 log_err_addr("proxy_protocol: not enough "
2095                                         "buffer size to read PROXYv2 header", "",
2096                                         &c->repinfo.remote_addr,
2097                                         c->repinfo.remote_addrlen);
2098                                 return 0;
2099                         }
2100                         verbose(VERB_ALGO, "proxy_protocol: reading fixed "
2101                                 "part of PROXYv2 header (len %lu)",
2102                                 (unsigned long)want_read_size);
2103                         current_read_size = want_read_size;
2104                         if(c->tcp_byte_count < current_read_size) {
2105                                 r = recv(fd, (void*)sldns_buffer_at(c->buffer,
2106                                         c->tcp_byte_count),
2107                                         current_read_size-c->tcp_byte_count, MSG_DONTWAIT);
2108                                 if(r == 0) {
2109                                         if(c->tcp_req_info)
2110                                                 return tcp_req_info_handle_read_close(c->tcp_req_info);
2111                                         return 0;
2112                                 } else if(r == -1) {
2113                                         goto recv_error_initial;
2114                                 }
2115                                 c->tcp_byte_count += r;
2116                                 if(c->tcp_byte_count != current_read_size) return 1;
2117                                 c->pp2_header_state = pp2_header_init;
2118                         }
2119                 }
2120                 if(c->pp2_header_state == pp2_header_init) {
2121                         header = pp2_read_header(c->buffer);
2122                         if(!header) {
2123                                 log_err("proxy_protocol: could not parse "
2124                                         "PROXYv2 header");
2125                                 return 0;
2126                         }
2127                         want_read_size = ntohs(header->len);
2128                         if(sldns_buffer_remaining(c->buffer) <
2129                                 PP2_HEADER_SIZE + want_read_size) {
2130                                 log_err_addr("proxy_protocol: not enough "
2131                                         "buffer size to read PROXYv2 header", "",
2132                                         &c->repinfo.remote_addr,
2133                                         c->repinfo.remote_addrlen);
2134                                 return 0;
2135                         }
2136                         verbose(VERB_ALGO, "proxy_protocol: reading variable "
2137                                 "part of PROXYv2 header (len %lu)",
2138                                 (unsigned long)want_read_size);
2139                         current_read_size = PP2_HEADER_SIZE + want_read_size;
2140                         if(want_read_size == 0) {
2141                                 /* nothing more to read; header is complete */
2142                                 c->pp2_header_state = pp2_header_done;
2143                         } else if(c->tcp_byte_count < current_read_size) {
2144                                 r = recv(fd, (void*)sldns_buffer_at(c->buffer,
2145                                         c->tcp_byte_count),
2146                                         current_read_size-c->tcp_byte_count, MSG_DONTWAIT);
2147                                 if(r == 0) {
2148                                         if(c->tcp_req_info)
2149                                                 return tcp_req_info_handle_read_close(c->tcp_req_info);
2150                                         return 0;
2151                                 } else if(r == -1) {
2152                                         goto recv_error;
2153                                 }
2154                                 c->tcp_byte_count += r;
2155                                 if(c->tcp_byte_count != current_read_size) return 1;
2156                                 c->pp2_header_state = pp2_header_done;
2157                         }
2158                 }
2159                 if(c->pp2_header_state != pp2_header_done || !header) {
2160                         log_err_addr("proxy_protocol: wrong state for the "
2161                                 "PROXYv2 header", "", &c->repinfo.remote_addr,
2162                                 c->repinfo.remote_addrlen);
2163                         return 0;
2164                 }
2165                 if(!consume_pp2_header(c->buffer, &c->repinfo, 1)) {
2166                         log_err_addr("proxy_protocol: could not consume "
2167                                 "PROXYv2 header", "", &c->repinfo.remote_addr,
2168                                 c->repinfo.remote_addrlen);
2169                         return 0;
2170                 }
2171                 verbose(VERB_ALGO, "proxy_protocol: successful read of "
2172                         "PROXYv2 header");
2173                 /* Clear and reset the buffer to read the following
2174                     * DNS packet(s). */
2175                 sldns_buffer_clear(c->buffer);
2176                 c->tcp_byte_count = 0;
2177                 return 1;
2178         }
2179
2180         if(c->tcp_byte_count < sizeof(uint16_t)) {
2181                 /* read length bytes */
2182                 r = recv(fd,(void*)sldns_buffer_at(c->buffer,c->tcp_byte_count),
2183                         sizeof(uint16_t)-c->tcp_byte_count, MSG_DONTWAIT);
2184                 if(r == 0) {
2185                         if(c->tcp_req_info)
2186                                 return tcp_req_info_handle_read_close(c->tcp_req_info);
2187                         return 0;
2188                 } else if(r == -1) {
2189                         if(c->pp2_enabled) goto recv_error;
2190                         goto recv_error_initial;
2191                 }
2192                 c->tcp_byte_count += r;
2193                 if(c->tcp_byte_count != sizeof(uint16_t))
2194                         return 1;
2195                 if(sldns_buffer_read_u16_at(c->buffer, 0) >
2196                         sldns_buffer_capacity(c->buffer)) {
2197                         verbose(VERB_QUERY, "tcp: dropped larger than buffer");
2198                         return 0;
2199                 }
2200                 sldns_buffer_set_limit(c->buffer,
2201                         sldns_buffer_read_u16_at(c->buffer, 0));
2202                 if(!short_ok &&
2203                         sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
2204                         verbose(VERB_QUERY, "tcp: dropped bogus too short.");
2205                         return 0;
2206                 }
2207                 verbose(VERB_ALGO, "Reading tcp query of length %d",
2208                         (int)sldns_buffer_limit(c->buffer));
2209         }
2210
2211         if(sldns_buffer_remaining(c->buffer) == 0)
2212                 log_err("in comm_point_tcp_handle_read buffer_remaining is "
2213                         "not > 0 as expected, continuing with (harmless) 0 "
2214                         "length recv");
2215         r = recv(fd, (void*)sldns_buffer_current(c->buffer),
2216                 sldns_buffer_remaining(c->buffer), MSG_DONTWAIT);
2217         if(r == 0) {
2218                 if(c->tcp_req_info)
2219                         return tcp_req_info_handle_read_close(c->tcp_req_info);
2220                 return 0;
2221         } else if(r == -1) {
2222                 goto recv_error;
2223         }
2224         sldns_buffer_skip(c->buffer, r);
2225         if(sldns_buffer_remaining(c->buffer) <= 0) {
2226                 tcp_callback_reader(c);
2227         }
2228         return 1;
2229
2230 recv_error_initial:
2231         recv_initial = 1;
2232 recv_error:
2233 #ifndef USE_WINSOCK
2234         if(errno == EINTR || errno == EAGAIN)
2235                 return 1;
2236         if(recv_initial) {
2237 #ifdef ECONNRESET
2238                 if(errno == ECONNRESET && verbosity < 2)
2239                         return 0; /* silence reset by peer */
2240 #endif
2241 #ifdef ECONNREFUSED
2242                 if(errno == ECONNREFUSED && verbosity < 2)
2243                         return 0; /* silence reset by peer */
2244 #endif
2245 #ifdef ENETUNREACH
2246                 if(errno == ENETUNREACH && verbosity < 2)
2247                         return 0; /* silence it */
2248 #endif
2249 #ifdef EHOSTDOWN
2250                 if(errno == EHOSTDOWN && verbosity < 2)
2251                         return 0; /* silence it */
2252 #endif
2253 #ifdef EHOSTUNREACH
2254                 if(errno == EHOSTUNREACH && verbosity < 2)
2255                         return 0; /* silence it */
2256 #endif
2257 #ifdef ENETDOWN
2258                 if(errno == ENETDOWN && verbosity < 2)
2259                         return 0; /* silence it */
2260 #endif
2261 #ifdef EACCES
2262                 if(errno == EACCES && verbosity < 2)
2263                         return 0; /* silence it */
2264 #endif
2265 #ifdef ENOTCONN
2266                 if(errno == ENOTCONN) {
2267                         log_err_addr("read (in tcp s) failed and this "
2268                                 "could be because TCP Fast Open is "
2269                                 "enabled [--disable-tfo-client "
2270                                 "--disable-tfo-server] but does not "
2271                                 "work", sock_strerror(errno),
2272                                 &c->repinfo.remote_addr,
2273                                 c->repinfo.remote_addrlen);
2274                         return 0;
2275                 }
2276 #endif
2277         }
2278 #else /* USE_WINSOCK */
2279         if(recv_initial) {
2280                 if(WSAGetLastError() == WSAECONNREFUSED && verbosity < 2)
2281                         return 0;
2282                 if(WSAGetLastError() == WSAEHOSTDOWN && verbosity < 2)
2283                         return 0;
2284                 if(WSAGetLastError() == WSAEHOSTUNREACH && verbosity < 2)
2285                         return 0;
2286                 if(WSAGetLastError() == WSAENETDOWN && verbosity < 2)
2287                         return 0;
2288                 if(WSAGetLastError() == WSAENETUNREACH && verbosity < 2)
2289                         return 0;
2290         }
2291         if(WSAGetLastError() == WSAECONNRESET)
2292                 return 0;
2293         if(WSAGetLastError() == WSAEINPROGRESS)
2294                 return 1;
2295         if(WSAGetLastError() == WSAEWOULDBLOCK) {
2296                 ub_winsock_tcp_wouldblock(c->ev->ev,
2297                         UB_EV_READ);
2298                 return 1;
2299         }
2300 #endif
2301         log_err_addr("read (in tcp s)", sock_strerror(errno),
2302                 &c->repinfo.remote_addr, c->repinfo.remote_addrlen);
2303         return 0;
2304 }
2305
2306 /**
2307  * Handle tcp writing callback.
2308  * @param fd: file descriptor of socket.
2309  * @param c: comm point to write buffer out of.
2310  * @return: 0 on error
2311  */
2312 static int
2313 comm_point_tcp_handle_write(int fd, struct comm_point* c)
2314 {
2315         ssize_t r;
2316         struct sldns_buffer *buffer;
2317         log_assert(c->type == comm_tcp);
2318 #ifdef USE_DNSCRYPT
2319         buffer = c->dnscrypt_buffer;
2320 #else
2321         buffer = c->buffer;
2322 #endif
2323         if(c->tcp_is_reading && !c->ssl && !c->tcp_write_and_read)
2324                 return 0;
2325         log_assert(fd != -1);
2326         if(((!c->tcp_write_and_read && c->tcp_byte_count == 0) || (c->tcp_write_and_read && c->tcp_write_byte_count == 0)) && c->tcp_check_nb_connect) {
2327                 /* check for pending error from nonblocking connect */
2328                 /* from Stevens, unix network programming, vol1, 3rd ed, p450*/
2329                 int error = 0;
2330                 socklen_t len = (socklen_t)sizeof(error);
2331                 if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
2332                         &len) < 0){
2333 #ifndef USE_WINSOCK
2334                         error = errno; /* on solaris errno is error */
2335 #else /* USE_WINSOCK */
2336                         error = WSAGetLastError();
2337 #endif
2338                 }
2339 #ifndef USE_WINSOCK
2340 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
2341                 if(error == EINPROGRESS || error == EWOULDBLOCK)
2342                         return 1; /* try again later */
2343                 else
2344 #endif
2345                 if(error != 0 && verbosity < 2)
2346                         return 0; /* silence lots of chatter in the logs */
2347                 else if(error != 0) {
2348                         log_err_addr("tcp connect", strerror(error),
2349                                 &c->repinfo.remote_addr,
2350                                 c->repinfo.remote_addrlen);
2351 #else /* USE_WINSOCK */
2352                 /* examine error */
2353                 if(error == WSAEINPROGRESS)
2354                         return 1;
2355                 else if(error == WSAEWOULDBLOCK) {
2356                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2357                         return 1;
2358                 } else if(error != 0 && verbosity < 2)
2359                         return 0;
2360                 else if(error != 0) {
2361                         log_err_addr("tcp connect", wsa_strerror(error),
2362                                 &c->repinfo.remote_addr,
2363                                 c->repinfo.remote_addrlen);
2364 #endif /* USE_WINSOCK */
2365                         return 0;
2366                 }
2367         }
2368         if(c->ssl)
2369                 return ssl_handle_it(c, 1);
2370
2371 #ifdef USE_MSG_FASTOPEN
2372         /* Only try this on first use of a connection that uses tfo,
2373            otherwise fall through to normal write */
2374         /* Also, TFO support on WINDOWS not implemented at the moment */
2375         if(c->tcp_do_fastopen == 1) {
2376                 /* this form of sendmsg() does both a connect() and send() so need to
2377                    look for various flavours of error*/
2378                 uint16_t len = htons(c->tcp_write_and_read?c->tcp_write_pkt_len:sldns_buffer_limit(buffer));
2379                 struct msghdr msg;
2380                 struct iovec iov[2];
2381                 c->tcp_do_fastopen = 0;
2382                 memset(&msg, 0, sizeof(msg));
2383                 if(c->tcp_write_and_read) {
2384                         iov[0].iov_base = (uint8_t*)&len + c->tcp_write_byte_count;
2385                         iov[0].iov_len = sizeof(uint16_t) - c->tcp_write_byte_count;
2386                         iov[1].iov_base = c->tcp_write_pkt;
2387                         iov[1].iov_len = c->tcp_write_pkt_len;
2388                 } else {
2389                         iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
2390                         iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
2391                         iov[1].iov_base = sldns_buffer_begin(buffer);
2392                         iov[1].iov_len = sldns_buffer_limit(buffer);
2393                 }
2394                 log_assert(iov[0].iov_len > 0);
2395                 msg.msg_name = &c->repinfo.remote_addr;
2396                 msg.msg_namelen = c->repinfo.remote_addrlen;
2397                 msg.msg_iov = iov;
2398                 msg.msg_iovlen = 2;
2399                 r = sendmsg(fd, &msg, MSG_FASTOPEN);
2400                 if (r == -1) {
2401 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
2402                         /* Handshake is underway, maybe because no TFO cookie available.
2403                            Come back to write the message*/
2404                         if(errno == EINPROGRESS || errno == EWOULDBLOCK)
2405                                 return 1;
2406 #endif
2407                         if(errno == EINTR || errno == EAGAIN)
2408                                 return 1;
2409                         /* Not handling EISCONN here as shouldn't ever hit that case.*/
2410                         if(errno != EPIPE
2411 #ifdef EOPNOTSUPP
2412                                 /* if /proc/sys/net/ipv4/tcp_fastopen is
2413                                  * disabled on Linux, sendmsg may return
2414                                  * 'Operation not supported', if so
2415                                  * fallthrough to ordinary connect. */
2416                                 && errno != EOPNOTSUPP
2417 #endif
2418                                 && errno != 0) {
2419                                 if(verbosity < 2)
2420                                         return 0; /* silence lots of chatter in the logs */
2421                                 log_err_addr("tcp sendmsg", strerror(errno),
2422                                         &c->repinfo.remote_addr,
2423                                         c->repinfo.remote_addrlen);
2424                                 return 0;
2425                         }
2426                         verbose(VERB_ALGO, "tcp sendmsg for fastopen failed (with %s), try normal connect", strerror(errno));
2427                         /* fallthrough to nonFASTOPEN
2428                          * (MSG_FASTOPEN on Linux 3 produces EPIPE)
2429                          * we need to perform connect() */
2430                         if(connect(fd, (struct sockaddr *)&c->repinfo.remote_addr,
2431                                 c->repinfo.remote_addrlen) == -1) {
2432 #ifdef EINPROGRESS
2433                                 if(errno == EINPROGRESS)
2434                                         return 1; /* wait until connect done*/
2435 #endif
2436 #ifdef USE_WINSOCK
2437                                 if(WSAGetLastError() == WSAEINPROGRESS ||
2438                                         WSAGetLastError() == WSAEWOULDBLOCK)
2439                                         return 1; /* wait until connect done*/
2440 #endif
2441                                 if(tcp_connect_errno_needs_log(
2442                                         (struct sockaddr *)&c->repinfo.remote_addr,
2443                                         c->repinfo.remote_addrlen)) {
2444                                         log_err_addr("outgoing tcp: connect after EPIPE for fastopen",
2445                                                 strerror(errno),
2446                                                 &c->repinfo.remote_addr,
2447                                                 c->repinfo.remote_addrlen);
2448                                 }
2449                                 return 0;
2450                         }
2451
2452                 } else {
2453                         if(c->tcp_write_and_read) {
2454                                 c->tcp_write_byte_count += r;
2455                                 if(c->tcp_write_byte_count < sizeof(uint16_t))
2456                                         return 1;
2457                         } else {
2458                                 c->tcp_byte_count += r;
2459                                 if(c->tcp_byte_count < sizeof(uint16_t))
2460                                         return 1;
2461                                 sldns_buffer_set_position(buffer, c->tcp_byte_count -
2462                                         sizeof(uint16_t));
2463                         }
2464                         if((!c->tcp_write_and_read && sldns_buffer_remaining(buffer) == 0) || (c->tcp_write_and_read && c->tcp_write_byte_count == c->tcp_write_pkt_len + 2)) {
2465                                 tcp_callback_writer(c);
2466                                 return 1;
2467                         }
2468                 }
2469         }
2470 #endif /* USE_MSG_FASTOPEN */
2471
2472         if((c->tcp_write_and_read?c->tcp_write_byte_count:c->tcp_byte_count) < sizeof(uint16_t)) {
2473                 uint16_t len = htons(c->tcp_write_and_read?c->tcp_write_pkt_len:sldns_buffer_limit(buffer));
2474 #ifdef HAVE_WRITEV
2475                 struct iovec iov[2];
2476                 if(c->tcp_write_and_read) {
2477                         iov[0].iov_base = (uint8_t*)&len + c->tcp_write_byte_count;
2478                         iov[0].iov_len = sizeof(uint16_t) - c->tcp_write_byte_count;
2479                         iov[1].iov_base = c->tcp_write_pkt;
2480                         iov[1].iov_len = c->tcp_write_pkt_len;
2481                 } else {
2482                         iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
2483                         iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
2484                         iov[1].iov_base = sldns_buffer_begin(buffer);
2485                         iov[1].iov_len = sldns_buffer_limit(buffer);
2486                 }
2487                 log_assert(iov[0].iov_len > 0);
2488                 r = writev(fd, iov, 2);
2489 #else /* HAVE_WRITEV */
2490                 if(c->tcp_write_and_read) {
2491                         r = send(fd, (void*)(((uint8_t*)&len)+c->tcp_write_byte_count),
2492                                 sizeof(uint16_t)-c->tcp_write_byte_count, 0);
2493                 } else {
2494                         r = send(fd, (void*)(((uint8_t*)&len)+c->tcp_byte_count),
2495                                 sizeof(uint16_t)-c->tcp_byte_count, 0);
2496                 }
2497 #endif /* HAVE_WRITEV */
2498                 if(r == -1) {
2499 #ifndef USE_WINSOCK
2500 #  ifdef EPIPE
2501                         if(errno == EPIPE && verbosity < 2)
2502                                 return 0; /* silence 'broken pipe' */
2503   #endif
2504                         if(errno == EINTR || errno == EAGAIN)
2505                                 return 1;
2506 #ifdef ECONNRESET
2507                         if(errno == ECONNRESET && verbosity < 2)
2508                                 return 0; /* silence reset by peer */
2509 #endif
2510 #  ifdef HAVE_WRITEV
2511                         log_err_addr("tcp writev", strerror(errno),
2512                                 &c->repinfo.remote_addr,
2513                                 c->repinfo.remote_addrlen);
2514 #  else /* HAVE_WRITEV */
2515                         log_err_addr("tcp send s", strerror(errno),
2516                                 &c->repinfo.remote_addr,
2517                                 c->repinfo.remote_addrlen);
2518 #  endif /* HAVE_WRITEV */
2519 #else
2520                         if(WSAGetLastError() == WSAENOTCONN)
2521                                 return 1;
2522                         if(WSAGetLastError() == WSAEINPROGRESS)
2523                                 return 1;
2524                         if(WSAGetLastError() == WSAEWOULDBLOCK) {
2525                                 ub_winsock_tcp_wouldblock(c->ev->ev,
2526                                         UB_EV_WRITE);
2527                                 return 1;
2528                         }
2529                         if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
2530                                 return 0; /* silence reset by peer */
2531                         log_err_addr("tcp send s",
2532                                 wsa_strerror(WSAGetLastError()),
2533                                 &c->repinfo.remote_addr,
2534                                 c->repinfo.remote_addrlen);
2535 #endif
2536                         return 0;
2537                 }
2538                 if(c->tcp_write_and_read) {
2539                         c->tcp_write_byte_count += r;
2540                         if(c->tcp_write_byte_count < sizeof(uint16_t))
2541                                 return 1;
2542                 } else {
2543                         c->tcp_byte_count += r;
2544                         if(c->tcp_byte_count < sizeof(uint16_t))
2545                                 return 1;
2546                         sldns_buffer_set_position(buffer, c->tcp_byte_count -
2547                                 sizeof(uint16_t));
2548                 }
2549                 if((!c->tcp_write_and_read && sldns_buffer_remaining(buffer) == 0) || (c->tcp_write_and_read && c->tcp_write_byte_count == c->tcp_write_pkt_len + 2)) {
2550                         tcp_callback_writer(c);
2551                         return 1;
2552                 }
2553         }
2554         log_assert(c->tcp_write_and_read || sldns_buffer_remaining(buffer) > 0);
2555         log_assert(!c->tcp_write_and_read || c->tcp_write_byte_count < c->tcp_write_pkt_len + 2);
2556         if(c->tcp_write_and_read) {
2557                 r = send(fd, (void*)(c->tcp_write_pkt + c->tcp_write_byte_count - 2),
2558                         c->tcp_write_pkt_len + 2 - c->tcp_write_byte_count, 0);
2559         } else {
2560                 r = send(fd, (void*)sldns_buffer_current(buffer),
2561                         sldns_buffer_remaining(buffer), 0);
2562         }
2563         if(r == -1) {
2564 #ifndef USE_WINSOCK
2565                 if(errno == EINTR || errno == EAGAIN)
2566                         return 1;
2567 #ifdef ECONNRESET
2568                 if(errno == ECONNRESET && verbosity < 2)
2569                         return 0; /* silence reset by peer */
2570 #endif
2571 #else
2572                 if(WSAGetLastError() == WSAEINPROGRESS)
2573                         return 1;
2574                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
2575                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
2576                         return 1;
2577                 }
2578                 if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
2579                         return 0; /* silence reset by peer */
2580 #endif
2581                 log_err_addr("tcp send r", sock_strerror(errno),
2582                         &c->repinfo.remote_addr,
2583                         c->repinfo.remote_addrlen);
2584                 return 0;
2585         }
2586         if(c->tcp_write_and_read) {
2587                 c->tcp_write_byte_count += r;
2588         } else {
2589                 sldns_buffer_skip(buffer, r);
2590         }
2591
2592         if((!c->tcp_write_and_read && sldns_buffer_remaining(buffer) == 0) || (c->tcp_write_and_read && c->tcp_write_byte_count == c->tcp_write_pkt_len + 2)) {
2593                 tcp_callback_writer(c);
2594         }
2595
2596         return 1;
2597 }
2598
2599 /** read again to drain buffers when there could be more to read, returns 0
2600  * on failure which means the comm point is closed. */
2601 static int
2602 tcp_req_info_read_again(int fd, struct comm_point* c)
2603 {
2604         while(c->tcp_req_info->read_again) {
2605                 int r;
2606                 c->tcp_req_info->read_again = 0;
2607                 if(c->tcp_is_reading)
2608                         r = comm_point_tcp_handle_read(fd, c, 0);
2609                 else    r = comm_point_tcp_handle_write(fd, c);
2610                 if(!r) {
2611                         reclaim_tcp_handler(c);
2612                         if(!c->tcp_do_close) {
2613                                 fptr_ok(fptr_whitelist_comm_point(
2614                                         c->callback));
2615                                 (void)(*c->callback)(c, c->cb_arg,
2616                                         NETEVENT_CLOSED, NULL);
2617                         }
2618                         return 0;
2619                 }
2620         }
2621         return 1;
2622 }
2623
2624 /** read again to drain buffers when there could be more to read */
2625 static void
2626 tcp_more_read_again(int fd, struct comm_point* c)
2627 {
2628         /* if the packet is done, but another one could be waiting on
2629          * the connection, the callback signals this, and we try again */
2630         /* this continues until the read routines get EAGAIN or so,
2631          * and thus does not call the callback, and the bool is 0 */
2632         int* moreread = c->tcp_more_read_again;
2633         while(moreread && *moreread) {
2634                 *moreread = 0;
2635                 if(!comm_point_tcp_handle_read(fd, c, 0)) {
2636                         reclaim_tcp_handler(c);
2637                         if(!c->tcp_do_close) {
2638                                 fptr_ok(fptr_whitelist_comm_point(
2639                                         c->callback));
2640                                 (void)(*c->callback)(c, c->cb_arg,
2641                                         NETEVENT_CLOSED, NULL);
2642                         }
2643                         return;
2644                 }
2645         }
2646 }
2647
2648 /** write again to fill up when there could be more to write */
2649 static void
2650 tcp_more_write_again(int fd, struct comm_point* c)
2651 {
2652         /* if the packet is done, but another is waiting to be written,
2653          * the callback signals it and we try again. */
2654         /* this continues until the write routines get EAGAIN or so,
2655          * and thus does not call the callback, and the bool is 0 */
2656         int* morewrite = c->tcp_more_write_again;
2657         while(morewrite && *morewrite) {
2658                 *morewrite = 0;
2659                 if(!comm_point_tcp_handle_write(fd, c)) {
2660                         reclaim_tcp_handler(c);
2661                         if(!c->tcp_do_close) {
2662                                 fptr_ok(fptr_whitelist_comm_point(
2663                                         c->callback));
2664                                 (void)(*c->callback)(c, c->cb_arg,
2665                                         NETEVENT_CLOSED, NULL);
2666                         }
2667                         return;
2668                 }
2669         }
2670 }
2671
2672 void
2673 comm_point_tcp_handle_callback(int fd, short event, void* arg)
2674 {
2675         struct comm_point* c = (struct comm_point*)arg;
2676         log_assert(c->type == comm_tcp);
2677         ub_comm_base_now(c->ev->base);
2678
2679         if(c->fd == -1 || c->fd != fd)
2680                 return; /* duplicate event, but commpoint closed. */
2681
2682 #ifdef USE_DNSCRYPT
2683         /* Initialize if this is a dnscrypt socket */
2684         if(c->tcp_parent) {
2685                 c->dnscrypt = c->tcp_parent->dnscrypt;
2686         }
2687         if(c->dnscrypt && c->dnscrypt_buffer == c->buffer) {
2688                 c->dnscrypt_buffer = sldns_buffer_new(sldns_buffer_capacity(c->buffer));
2689                 if(!c->dnscrypt_buffer) {
2690                         log_err("Could not allocate dnscrypt buffer");
2691                         reclaim_tcp_handler(c);
2692                         if(!c->tcp_do_close) {
2693                                 fptr_ok(fptr_whitelist_comm_point(
2694                                         c->callback));
2695                                 (void)(*c->callback)(c, c->cb_arg,
2696                                         NETEVENT_CLOSED, NULL);
2697                         }
2698                         return;
2699                 }
2700         }
2701 #endif
2702
2703         if(event&UB_EV_TIMEOUT) {
2704                 verbose(VERB_QUERY, "tcp took too long, dropped");
2705                 reclaim_tcp_handler(c);
2706                 if(!c->tcp_do_close) {
2707                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2708                         (void)(*c->callback)(c, c->cb_arg,
2709                                 NETEVENT_TIMEOUT, NULL);
2710                 }
2711                 return;
2712         }
2713         if(event&UB_EV_READ
2714 #ifdef USE_MSG_FASTOPEN
2715                 && !(c->tcp_do_fastopen && (event&UB_EV_WRITE))
2716 #endif
2717                 ) {
2718                 int has_tcpq = (c->tcp_req_info != NULL);
2719                 int* moreread = c->tcp_more_read_again;
2720                 if(!comm_point_tcp_handle_read(fd, c, 0)) {
2721                         reclaim_tcp_handler(c);
2722                         if(!c->tcp_do_close) {
2723                                 fptr_ok(fptr_whitelist_comm_point(
2724                                         c->callback));
2725                                 (void)(*c->callback)(c, c->cb_arg,
2726                                         NETEVENT_CLOSED, NULL);
2727                         }
2728                         return;
2729                 }
2730                 if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again) {
2731                         if(!tcp_req_info_read_again(fd, c))
2732                                 return;
2733                 }
2734                 if(moreread && *moreread)
2735                         tcp_more_read_again(fd, c);
2736                 return;
2737         }
2738         if(event&UB_EV_WRITE) {
2739                 int has_tcpq = (c->tcp_req_info != NULL);
2740                 int* morewrite = c->tcp_more_write_again;
2741                 if(!comm_point_tcp_handle_write(fd, c)) {
2742                         reclaim_tcp_handler(c);
2743                         if(!c->tcp_do_close) {
2744                                 fptr_ok(fptr_whitelist_comm_point(
2745                                         c->callback));
2746                                 (void)(*c->callback)(c, c->cb_arg,
2747                                         NETEVENT_CLOSED, NULL);
2748                         }
2749                         return;
2750                 }
2751                 if(has_tcpq && c->tcp_req_info && c->tcp_req_info->read_again) {
2752                         if(!tcp_req_info_read_again(fd, c))
2753                                 return;
2754                 }
2755                 if(morewrite && *morewrite)
2756                         tcp_more_write_again(fd, c);
2757                 return;
2758         }
2759         log_err("Ignored event %d for tcphdl.", event);
2760 }
2761
2762 /** Make http handler free for next assignment */
2763 static void
2764 reclaim_http_handler(struct comm_point* c)
2765 {
2766         log_assert(c->type == comm_http);
2767         if(c->ssl) {
2768 #ifdef HAVE_SSL
2769                 SSL_shutdown(c->ssl);
2770                 SSL_free(c->ssl);
2771                 c->ssl = NULL;
2772 #endif
2773         }
2774         comm_point_close(c);
2775         if(c->tcp_parent) {
2776                 if(c != c->tcp_parent->tcp_free) {
2777                         c->tcp_parent->cur_tcp_count--;
2778                         c->tcp_free = c->tcp_parent->tcp_free;
2779                         c->tcp_parent->tcp_free = c;
2780                 }
2781                 if(!c->tcp_free) {
2782                         /* re-enable listening on accept socket */
2783                         comm_point_start_listening(c->tcp_parent, -1, -1);
2784                 }
2785         }
2786 }
2787
2788 /** read more data for http (with ssl) */
2789 static int
2790 ssl_http_read_more(struct comm_point* c)
2791 {
2792 #ifdef HAVE_SSL
2793         int r;
2794         log_assert(sldns_buffer_remaining(c->buffer) > 0);
2795         ERR_clear_error();
2796         r = SSL_read(c->ssl, (void*)sldns_buffer_current(c->buffer),
2797                 (int)sldns_buffer_remaining(c->buffer));
2798         if(r <= 0) {
2799                 int want = SSL_get_error(c->ssl, r);
2800                 if(want == SSL_ERROR_ZERO_RETURN) {
2801                         return 0; /* shutdown, closed */
2802                 } else if(want == SSL_ERROR_WANT_READ) {
2803                         return 1; /* read more later */
2804                 } else if(want == SSL_ERROR_WANT_WRITE) {
2805                         c->ssl_shake_state = comm_ssl_shake_hs_write;
2806                         comm_point_listen_for_rw(c, 0, 1);
2807                         return 1;
2808                 } else if(want == SSL_ERROR_SYSCALL) {
2809 #ifdef ECONNRESET
2810                         if(errno == ECONNRESET && verbosity < 2)
2811                                 return 0; /* silence reset by peer */
2812 #endif
2813                         if(errno != 0)
2814                                 log_err("SSL_read syscall: %s",
2815                                         strerror(errno));
2816                         return 0;
2817                 }
2818                 log_crypto_err("could not SSL_read");
2819                 return 0;
2820         }
2821         verbose(VERB_ALGO, "ssl http read more skip to %d + %d",
2822                 (int)sldns_buffer_position(c->buffer), (int)r);
2823         sldns_buffer_skip(c->buffer, (ssize_t)r);
2824         return 1;
2825 #else
2826         (void)c;
2827         return 0;
2828 #endif /* HAVE_SSL */
2829 }
2830
2831 /** read more data for http */
2832 static int
2833 http_read_more(int fd, struct comm_point* c)
2834 {
2835         ssize_t r;
2836         log_assert(sldns_buffer_remaining(c->buffer) > 0);
2837         r = recv(fd, (void*)sldns_buffer_current(c->buffer),
2838                 sldns_buffer_remaining(c->buffer), MSG_DONTWAIT);
2839         if(r == 0) {
2840                 return 0;
2841         } else if(r == -1) {
2842 #ifndef USE_WINSOCK
2843                 if(errno == EINTR || errno == EAGAIN)
2844                         return 1;
2845 #else /* USE_WINSOCK */
2846                 if(WSAGetLastError() == WSAECONNRESET)
2847                         return 0;
2848                 if(WSAGetLastError() == WSAEINPROGRESS)
2849                         return 1;
2850                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
2851                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
2852                         return 1;
2853                 }
2854 #endif
2855                 log_err_addr("read (in http r)", sock_strerror(errno),
2856                         &c->repinfo.remote_addr, c->repinfo.remote_addrlen);
2857                 return 0;
2858         }
2859         verbose(VERB_ALGO, "http read more skip to %d + %d",
2860                 (int)sldns_buffer_position(c->buffer), (int)r);
2861         sldns_buffer_skip(c->buffer, r);
2862         return 1;
2863 }
2864
2865 /** return true if http header has been read (one line complete) */
2866 static int
2867 http_header_done(sldns_buffer* buf)
2868 {
2869         size_t i;
2870         for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
2871                 /* there was a \r before the \n, but we ignore that */
2872                 if((char)sldns_buffer_read_u8_at(buf, i) == '\n')
2873                         return 1;
2874         }
2875         return 0;
2876 }
2877
2878 /** return character string into buffer for header line, moves buffer
2879  * past that line and puts zero terminator into linefeed-newline */
2880 static char*
2881 http_header_line(sldns_buffer* buf)
2882 {
2883         char* result = (char*)sldns_buffer_current(buf);
2884         size_t i;
2885         for(i=sldns_buffer_position(buf); i<sldns_buffer_limit(buf); i++) {
2886                 /* terminate the string on the \r */
2887                 if((char)sldns_buffer_read_u8_at(buf, i) == '\r')
2888                         sldns_buffer_write_u8_at(buf, i, 0);
2889                 /* terminate on the \n and skip past the it and done */
2890                 if((char)sldns_buffer_read_u8_at(buf, i) == '\n') {
2891                         sldns_buffer_write_u8_at(buf, i, 0);
2892                         sldns_buffer_set_position(buf, i+1);
2893                         return result;
2894                 }
2895         }
2896         return NULL;
2897 }
2898
2899 /** move unread buffer to start and clear rest for putting the rest into it */
2900 static void
2901 http_moveover_buffer(sldns_buffer* buf)
2902 {
2903         size_t pos = sldns_buffer_position(buf);
2904         size_t len = sldns_buffer_remaining(buf);
2905         sldns_buffer_clear(buf);
2906         memmove(sldns_buffer_begin(buf), sldns_buffer_at(buf, pos), len);
2907         sldns_buffer_set_position(buf, len);
2908 }
2909
2910 /** a http header is complete, process it */
2911 static int
2912 http_process_initial_header(struct comm_point* c)
2913 {
2914         char* line = http_header_line(c->buffer);
2915         if(!line) return 1;
2916         verbose(VERB_ALGO, "http header: %s", line);
2917         if(strncasecmp(line, "HTTP/1.1 ", 9) == 0) {
2918                 /* check returncode */
2919                 if(line[9] != '2') {
2920                         verbose(VERB_ALGO, "http bad status %s", line+9);
2921                         return 0;
2922                 }
2923         } else if(strncasecmp(line, "Content-Length: ", 16) == 0) {
2924                 if(!c->http_is_chunked)
2925                         c->tcp_byte_count = (size_t)atoi(line+16);
2926         } else if(strncasecmp(line, "Transfer-Encoding: chunked", 19+7) == 0) {
2927                 c->tcp_byte_count = 0;
2928                 c->http_is_chunked = 1;
2929         } else if(line[0] == 0) {
2930                 /* end of initial headers */
2931                 c->http_in_headers = 0;
2932                 if(c->http_is_chunked)
2933                         c->http_in_chunk_headers = 1;
2934                 /* remove header text from front of buffer
2935                  * the buffer is going to be used to return the data segment
2936                  * itself and we don't want the header to get returned
2937                  * prepended with it */
2938                 http_moveover_buffer(c->buffer);
2939                 sldns_buffer_flip(c->buffer);
2940                 return 1;
2941         }
2942         /* ignore other headers */
2943         return 1;
2944 }
2945
2946 /** a chunk header is complete, process it, return 0=fail, 1=continue next
2947  * header line, 2=done with chunked transfer*/
2948 static int
2949 http_process_chunk_header(struct comm_point* c)
2950 {
2951         char* line = http_header_line(c->buffer);
2952         if(!line) return 1;
2953         if(c->http_in_chunk_headers == 3) {
2954                 verbose(VERB_ALGO, "http chunk trailer: %s", line);
2955                 /* are we done ? */
2956                 if(line[0] == 0 && c->tcp_byte_count == 0) {
2957                         /* callback of http reader when NETEVENT_DONE,
2958                          * end of data, with no data in buffer */
2959                         sldns_buffer_set_position(c->buffer, 0);
2960                         sldns_buffer_set_limit(c->buffer, 0);
2961                         fptr_ok(fptr_whitelist_comm_point(c->callback));
2962                         (void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
2963                         /* return that we are done */
2964                         return 2;
2965                 }
2966                 if(line[0] == 0) {
2967                         /* continue with header of the next chunk */
2968                         c->http_in_chunk_headers = 1;
2969                         /* remove header text from front of buffer */
2970                         http_moveover_buffer(c->buffer);
2971                         sldns_buffer_flip(c->buffer);
2972                         return 1;
2973                 }
2974                 /* ignore further trail headers */
2975                 return 1;
2976         }
2977         verbose(VERB_ALGO, "http chunk header: %s", line);
2978         if(c->http_in_chunk_headers == 1) {
2979                 /* read chunked start line */
2980                 char* end = NULL;
2981                 c->tcp_byte_count = (size_t)strtol(line, &end, 16);
2982                 if(end == line)
2983                         return 0;
2984                 c->http_in_chunk_headers = 0;
2985                 /* remove header text from front of buffer */
2986                 http_moveover_buffer(c->buffer);
2987                 sldns_buffer_flip(c->buffer);
2988                 if(c->tcp_byte_count == 0) {
2989                         /* done with chunks, process chunk_trailer lines */
2990                         c->http_in_chunk_headers = 3;
2991                 }
2992                 return 1;
2993         }
2994         /* ignore other headers */
2995         return 1;
2996 }
2997
2998 /** handle nonchunked data segment, 0=fail, 1=wait */
2999 static int
3000 http_nonchunk_segment(struct comm_point* c)
3001 {
3002         /* c->buffer at position..limit has new data we read in.
3003          * the buffer itself is full of nonchunked data.
3004          * we are looking to read tcp_byte_count more data
3005          * and then the transfer is done. */
3006         size_t remainbufferlen;
3007         size_t got_now = sldns_buffer_limit(c->buffer);
3008         if(c->tcp_byte_count <= got_now) {
3009                 /* done, this is the last data fragment */
3010                 c->http_stored = 0;
3011                 sldns_buffer_set_position(c->buffer, 0);
3012                 fptr_ok(fptr_whitelist_comm_point(c->callback));
3013                 (void)(*c->callback)(c, c->cb_arg, NETEVENT_DONE, NULL);
3014                 return 1;
3015         }
3016         /* if we have the buffer space,
3017          * read more data collected into the buffer */
3018         remainbufferlen = sldns_buffer_capacity(c->buffer) -
3019                 sldns_buffer_limit(c->buffer);
3020         if(remainbufferlen+got_now >= c->tcp_byte_count ||
3021                 remainbufferlen >= (size_t)(c->ssl?16384:2048)) {
3022                 size_t total = sldns_buffer_limit(c->buffer);
3023                 sldns_buffer_clear(c->buffer);
3024                 sldns_buffer_set_position(c->buffer, total);
3025                 c->http_stored = total;
3026                 /* return and wait to read more */
3027                 return 1;
3028         }
3029         /* call callback with this data amount, then
3030          * wait for more */
3031         c->tcp_byte_count -= got_now;
3032         c->http_stored = 0;
3033         sldns_buffer_set_position(c->buffer, 0);
3034         fptr_ok(fptr_whitelist_comm_point(c->callback));
3035         (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
3036         /* c->callback has to buffer_clear(c->buffer). */
3037         /* return and wait to read more */
3038         return 1;
3039 }
3040
3041 /** handle chunked data segment, return 0=fail, 1=wait, 2=process more */
3042 static int
3043 http_chunked_segment(struct comm_point* c)
3044 {
3045         /* the c->buffer has from position..limit new data we read. */
3046         /* the current chunk has length tcp_byte_count.
3047          * once we read that read more chunk headers.
3048          */
3049         size_t remainbufferlen;
3050         size_t got_now = sldns_buffer_limit(c->buffer) - c->http_stored;
3051         verbose(VERB_ALGO, "http_chunked_segment: got now %d, tcpbytcount %d, http_stored %d, buffer pos %d, buffer limit %d", (int)got_now, (int)c->tcp_byte_count, (int)c->http_stored, (int)sldns_buffer_position(c->buffer), (int)sldns_buffer_limit(c->buffer));
3052         if(c->tcp_byte_count <= got_now) {
3053                 /* the chunk has completed (with perhaps some extra data
3054                  * from next chunk header and next chunk) */
3055                 /* save too much info into temp buffer */
3056                 size_t fraglen;
3057                 struct comm_reply repinfo;
3058                 c->http_stored = 0;
3059                 sldns_buffer_skip(c->buffer, (ssize_t)c->tcp_byte_count);
3060                 sldns_buffer_clear(c->http_temp);
3061                 sldns_buffer_write(c->http_temp,
3062                         sldns_buffer_current(c->buffer),
3063                         sldns_buffer_remaining(c->buffer));
3064                 sldns_buffer_flip(c->http_temp);
3065
3066                 /* callback with this fragment */
3067                 fraglen = sldns_buffer_position(c->buffer);
3068                 sldns_buffer_set_position(c->buffer, 0);
3069                 sldns_buffer_set_limit(c->buffer, fraglen);
3070                 repinfo = c->repinfo;
3071                 fptr_ok(fptr_whitelist_comm_point(c->callback));
3072                 (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &repinfo);
3073                 /* c->callback has to buffer_clear(). */
3074
3075                 /* is commpoint deleted? */
3076                 if(!repinfo.c) {
3077                         return 1;
3078                 }
3079                 /* copy waiting info */
3080                 sldns_buffer_clear(c->buffer);
3081                 sldns_buffer_write(c->buffer,
3082                         sldns_buffer_begin(c->http_temp),
3083                         sldns_buffer_remaining(c->http_temp));
3084                 sldns_buffer_flip(c->buffer);
3085                 /* process end of chunk trailer header lines, until
3086                  * an empty line */
3087                 c->http_in_chunk_headers = 3;
3088                 /* process more data in buffer (if any) */
3089                 return 2;
3090         }
3091         c->tcp_byte_count -= got_now;
3092
3093         /* if we have the buffer space,
3094          * read more data collected into the buffer */
3095         remainbufferlen = sldns_buffer_capacity(c->buffer) -
3096                 sldns_buffer_limit(c->buffer);
3097         if(remainbufferlen >= c->tcp_byte_count ||
3098                 remainbufferlen >= 2048) {
3099                 size_t total = sldns_buffer_limit(c->buffer);
3100                 sldns_buffer_clear(c->buffer);
3101                 sldns_buffer_set_position(c->buffer, total);
3102                 c->http_stored = total;
3103                 /* return and wait to read more */
3104                 return 1;
3105         }
3106
3107         /* callback of http reader for a new part of the data */
3108         c->http_stored = 0;
3109         sldns_buffer_set_position(c->buffer, 0);
3110         fptr_ok(fptr_whitelist_comm_point(c->callback));
3111         (void)(*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, NULL);
3112         /* c->callback has to buffer_clear(c->buffer). */
3113         /* return and wait to read more */
3114         return 1;
3115 }
3116
3117 #ifdef HAVE_NGHTTP2
3118 /** Create new http2 session. Called when creating handling comm point. */
3119 static struct http2_session* http2_session_create(struct comm_point* c)
3120 {
3121         struct http2_session* session = calloc(1, sizeof(*session));
3122         if(!session) {
3123                 log_err("malloc failure while creating http2 session");
3124                 return NULL;
3125         }
3126         session->c = c;
3127
3128         return session;
3129 }
3130 #endif
3131
3132 /** Delete http2 session. After closing connection or on error */
3133 static void http2_session_delete(struct http2_session* h2_session)
3134 {
3135 #ifdef HAVE_NGHTTP2
3136         if(h2_session->callbacks)
3137                 nghttp2_session_callbacks_del(h2_session->callbacks);
3138         free(h2_session);
3139 #else
3140         (void)h2_session;
3141 #endif
3142 }
3143
3144 #ifdef HAVE_NGHTTP2
3145 struct http2_stream* http2_stream_create(int32_t stream_id)
3146 {
3147         struct http2_stream* h2_stream = calloc(1, sizeof(*h2_stream));
3148         if(!h2_stream) {
3149                 log_err("malloc failure while creating http2 stream");
3150                 return NULL;
3151         }
3152         h2_stream->stream_id = stream_id;
3153         return h2_stream;
3154 }
3155
3156 /** Delete http2 stream. After session delete or stream close callback */
3157 static void http2_stream_delete(struct http2_session* h2_session,
3158         struct http2_stream* h2_stream)
3159 {
3160         if(h2_stream->mesh_state) {
3161                 mesh_state_remove_reply(h2_stream->mesh, h2_stream->mesh_state,
3162                         h2_session->c);
3163                 h2_stream->mesh_state = NULL;
3164         }
3165         http2_req_stream_clear(h2_stream);
3166         free(h2_stream);
3167 }
3168 #endif
3169
3170 void http2_stream_add_meshstate(struct http2_stream* h2_stream,
3171         struct mesh_area* mesh, struct mesh_state* m)
3172 {
3173         h2_stream->mesh = mesh;
3174         h2_stream->mesh_state = m;
3175 }
3176
3177 /** delete http2 session server. After closing connection. */
3178 static void http2_session_server_delete(struct http2_session* h2_session)
3179 {
3180 #ifdef HAVE_NGHTTP2
3181         struct http2_stream* h2_stream, *next;
3182         nghttp2_session_del(h2_session->session); /* NULL input is fine */
3183         h2_session->session = NULL;
3184         for(h2_stream = h2_session->first_stream; h2_stream;) {
3185                 next = h2_stream->next;
3186                 http2_stream_delete(h2_session, h2_stream);
3187                 h2_stream = next;
3188         }
3189         h2_session->first_stream = NULL;
3190         h2_session->is_drop = 0;
3191         h2_session->postpone_drop = 0;
3192         h2_session->c->h2_stream = NULL;
3193 #endif
3194         (void)h2_session;
3195 }
3196
3197 #ifdef HAVE_NGHTTP2
3198 void http2_session_add_stream(struct http2_session* h2_session,
3199         struct http2_stream* h2_stream)
3200 {
3201         if(h2_session->first_stream)
3202                 h2_session->first_stream->prev = h2_stream;
3203         h2_stream->next = h2_session->first_stream;
3204         h2_session->first_stream = h2_stream;
3205 }
3206
3207 /** remove stream from session linked list. After stream close callback or
3208  * closing connection */
3209 static void http2_session_remove_stream(struct http2_session* h2_session,
3210         struct http2_stream* h2_stream)
3211 {
3212         if(h2_stream->prev)
3213                 h2_stream->prev->next = h2_stream->next;
3214         else
3215                 h2_session->first_stream = h2_stream->next;
3216         if(h2_stream->next)
3217                 h2_stream->next->prev = h2_stream->prev;
3218
3219 }
3220
3221 int http2_stream_close_cb(nghttp2_session* ATTR_UNUSED(session),
3222         int32_t stream_id, uint32_t ATTR_UNUSED(error_code), void* cb_arg)
3223 {
3224         struct http2_stream* h2_stream;
3225         struct http2_session* h2_session = (struct http2_session*)cb_arg;
3226         if(!(h2_stream = nghttp2_session_get_stream_user_data(
3227                 h2_session->session, stream_id))) {
3228                 return 0;
3229         }
3230         http2_session_remove_stream(h2_session, h2_stream);
3231         http2_stream_delete(h2_session, h2_stream);
3232         return 0;
3233 }
3234
3235 ssize_t http2_recv_cb(nghttp2_session* ATTR_UNUSED(session), uint8_t* buf,
3236         size_t len, int ATTR_UNUSED(flags), void* cb_arg)
3237 {
3238         struct http2_session* h2_session = (struct http2_session*)cb_arg;
3239         ssize_t ret;
3240
3241         log_assert(h2_session->c->type == comm_http);
3242         log_assert(h2_session->c->h2_session);
3243
3244 #ifdef HAVE_SSL
3245         if(h2_session->c->ssl) {
3246                 int r;
3247                 ERR_clear_error();
3248                 r = SSL_read(h2_session->c->ssl, buf, len);
3249                 if(r <= 0) {
3250                         int want = SSL_get_error(h2_session->c->ssl, r);
3251                         if(want == SSL_ERROR_ZERO_RETURN) {
3252                                 return NGHTTP2_ERR_EOF;
3253                         } else if(want == SSL_ERROR_WANT_READ) {
3254                                 return NGHTTP2_ERR_WOULDBLOCK;
3255                         } else if(want == SSL_ERROR_WANT_WRITE) {
3256                                 h2_session->c->ssl_shake_state = comm_ssl_shake_hs_write;
3257                                 comm_point_listen_for_rw(h2_session->c, 0, 1);
3258                                 return NGHTTP2_ERR_WOULDBLOCK;
3259                         } else if(want == SSL_ERROR_SYSCALL) {
3260 #ifdef ECONNRESET
3261                                 if(errno == ECONNRESET && verbosity < 2)
3262                                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3263 #endif
3264                                 if(errno != 0)
3265                                         log_err("SSL_read syscall: %s",
3266                                                 strerror(errno));
3267                                 return NGHTTP2_ERR_CALLBACK_FAILURE;
3268                         }
3269                         log_crypto_err("could not SSL_read");
3270                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3271                 }
3272                 return r;
3273         }
3274 #endif /* HAVE_SSL */
3275
3276         ret = recv(h2_session->c->fd, buf, len, MSG_DONTWAIT);
3277         if(ret == 0) {
3278                 return NGHTTP2_ERR_EOF;
3279         } else if(ret < 0) {
3280 #ifndef USE_WINSOCK
3281                 if(errno == EINTR || errno == EAGAIN)
3282                         return NGHTTP2_ERR_WOULDBLOCK;
3283 #ifdef ECONNRESET
3284                 if(errno == ECONNRESET && verbosity < 2)
3285                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3286 #endif
3287                 log_err_addr("could not http2 recv: %s", strerror(errno),
3288                         &h2_session->c->repinfo.remote_addr,
3289                         h2_session->c->repinfo.remote_addrlen);
3290 #else /* USE_WINSOCK */
3291                 if(WSAGetLastError() == WSAECONNRESET)
3292                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3293                 if(WSAGetLastError() == WSAEINPROGRESS)
3294                         return NGHTTP2_ERR_WOULDBLOCK;
3295                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
3296                         ub_winsock_tcp_wouldblock(h2_session->c->ev->ev,
3297                                 UB_EV_READ);
3298                         return NGHTTP2_ERR_WOULDBLOCK;
3299                 }
3300                 log_err_addr("could not http2 recv: %s",
3301                         wsa_strerror(WSAGetLastError()),
3302                         &h2_session->c->repinfo.remote_addr,
3303                         h2_session->c->repinfo.remote_addrlen);
3304 #endif
3305                 return NGHTTP2_ERR_CALLBACK_FAILURE;
3306         }
3307         return ret;
3308 }
3309 #endif /* HAVE_NGHTTP2 */
3310
3311 /** Handle http2 read */
3312 static int
3313 comm_point_http2_handle_read(int ATTR_UNUSED(fd), struct comm_point* c)
3314 {
3315 #ifdef HAVE_NGHTTP2
3316         int ret;
3317         log_assert(c->h2_session);
3318
3319         /* reading until recv cb returns NGHTTP2_ERR_WOULDBLOCK */
3320         ret = nghttp2_session_recv(c->h2_session->session);
3321         if(ret) {
3322                 if(ret != NGHTTP2_ERR_EOF &&
3323                         ret != NGHTTP2_ERR_CALLBACK_FAILURE) {
3324                         char a[256];
3325                         addr_to_str(&c->repinfo.remote_addr,
3326                                 c->repinfo.remote_addrlen, a, sizeof(a));
3327                         verbose(VERB_QUERY, "http2: session_recv from %s failed, "
3328                                 "error: %s", a, nghttp2_strerror(ret));
3329                 }
3330                 return 0;
3331         }
3332         if(nghttp2_session_want_write(c->h2_session->session)) {
3333                 c->tcp_is_reading = 0;
3334                 comm_point_stop_listening(c);
3335                 comm_point_start_listening(c, -1, adjusted_tcp_timeout(c));
3336         } else if(!nghttp2_session_want_read(c->h2_session->session))
3337                 return 0; /* connection can be closed */
3338         return 1;
3339 #else
3340         (void)c;
3341         return 0;
3342 #endif
3343 }
3344
3345 /**
3346  * Handle http reading callback.
3347  * @param fd: file descriptor of socket.
3348  * @param c: comm point to read from into buffer.
3349  * @return: 0 on error
3350  */
3351 static int
3352 comm_point_http_handle_read(int fd, struct comm_point* c)
3353 {
3354         log_assert(c->type == comm_http);
3355         log_assert(fd != -1);
3356
3357         /* if we are in ssl handshake, handle SSL handshake */
3358 #ifdef HAVE_SSL
3359         if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
3360                 if(!ssl_handshake(c))
3361                         return 0;
3362                 if(c->ssl_shake_state != comm_ssl_shake_none)
3363                         return 1;
3364         }
3365 #endif /* HAVE_SSL */
3366
3367         if(!c->tcp_is_reading)
3368                 return 1;
3369
3370         if(c->use_h2) {
3371                 return comm_point_http2_handle_read(fd, c);
3372         }
3373
3374         /* http version is <= http/1.1 */
3375
3376         if(c->http_min_version >= http_version_2) {
3377                 /* HTTP/2 failed, not allowed to use lower version. */
3378                 return 0;
3379         }
3380
3381         /* read more data */
3382         if(c->ssl) {
3383                 if(!ssl_http_read_more(c))
3384                         return 0;
3385         } else {
3386                 if(!http_read_more(fd, c))
3387                         return 0;
3388         }
3389
3390         if(c->http_stored >= sldns_buffer_position(c->buffer)) {
3391                 /* read did not work but we wanted more data, there is
3392                  * no bytes to process now. */
3393                 return 1;
3394         }
3395         sldns_buffer_flip(c->buffer);
3396         /* if we are partway in a segment of data, position us at the point
3397          * where we left off previously */
3398         if(c->http_stored < sldns_buffer_limit(c->buffer))
3399                 sldns_buffer_set_position(c->buffer, c->http_stored);
3400         else    sldns_buffer_set_position(c->buffer, sldns_buffer_limit(c->buffer));
3401
3402         while(sldns_buffer_remaining(c->buffer) > 0) {
3403                 /* Handle HTTP/1.x data */
3404                 /* if we are reading headers, read more headers */
3405                 if(c->http_in_headers || c->http_in_chunk_headers) {
3406                         /* if header is done, process the header */
3407                         if(!http_header_done(c->buffer)) {
3408                                 /* copy remaining data to front of buffer
3409                                  * and set rest for writing into it */
3410                                 http_moveover_buffer(c->buffer);
3411                                 /* return and wait to read more */
3412                                 return 1;
3413                         }
3414                         if(!c->http_in_chunk_headers) {
3415                                 /* process initial headers */
3416                                 if(!http_process_initial_header(c))
3417                                         return 0;
3418                         } else {
3419                                 /* process chunk headers */
3420                                 int r = http_process_chunk_header(c);
3421                                 if(r == 0) return 0;
3422                                 if(r == 2) return 1; /* done */
3423                                 /* r == 1, continue */
3424                         }
3425                         /* see if we have more to process */
3426                         continue;
3427                 }
3428
3429                 if(!c->http_is_chunked) {
3430                         /* if we are reading nonchunks, process that*/
3431                         return http_nonchunk_segment(c);
3432                 } else {
3433                         /* if we are reading chunks, read the chunk */
3434                         int r = http_chunked_segment(c);
3435                         if(r == 0) return 0;
3436                         if(r == 1) return 1;
3437                         continue;
3438                 }
3439         }
3440         /* broke out of the loop; could not process header instead need
3441          * to read more */
3442         /* moveover any remaining data and read more data */
3443         http_moveover_buffer(c->buffer);
3444         /* return and wait to read more */
3445         return 1;
3446 }
3447
3448 /** check pending connect for http */
3449 static int
3450 http_check_connect(int fd, struct comm_point* c)
3451 {
3452         /* check for pending error from nonblocking connect */
3453         /* from Stevens, unix network programming, vol1, 3rd ed, p450*/
3454         int error = 0;
3455         socklen_t len = (socklen_t)sizeof(error);
3456         if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,
3457                 &len) < 0){
3458 #ifndef USE_WINSOCK
3459                 error = errno; /* on solaris errno is error */
3460 #else /* USE_WINSOCK */
3461                 error = WSAGetLastError();
3462 #endif
3463         }
3464 #ifndef USE_WINSOCK
3465 #if defined(EINPROGRESS) && defined(EWOULDBLOCK)
3466         if(error == EINPROGRESS || error == EWOULDBLOCK)
3467                 return 1; /* try again later */
3468         else
3469 #endif
3470         if(error != 0 && verbosity < 2)
3471                 return 0; /* silence lots of chatter in the logs */
3472         else if(error != 0) {
3473                 log_err_addr("http connect", strerror(error),
3474                         &c->repinfo.remote_addr, c->repinfo.remote_addrlen);
3475 #else /* USE_WINSOCK */
3476         /* examine error */
3477         if(error == WSAEINPROGRESS)
3478                 return 1;
3479         else if(error == WSAEWOULDBLOCK) {
3480                 ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
3481                 return 1;
3482         } else if(error != 0 && verbosity < 2)
3483                 return 0;
3484         else if(error != 0) {
3485                 log_err_addr("http connect", wsa_strerror(error),
3486                         &c->repinfo.remote_addr, c->repinfo.remote_addrlen);
3487 #endif /* USE_WINSOCK */
3488                 return 0;
3489         }
3490         /* keep on processing this socket */
3491         return 2;
3492 }
3493
3494 /** write more data for http (with ssl) */
3495 static int
3496 ssl_http_write_more(struct comm_point* c)
3497 {
3498 #ifdef HAVE_SSL
3499         int r;
3500         log_assert(sldns_buffer_remaining(c->buffer) > 0);
3501         ERR_clear_error();
3502         r = SSL_write(c->ssl, (void*)sldns_buffer_current(c->buffer),
3503                 (int)sldns_buffer_remaining(c->buffer));
3504         if(r <= 0) {
3505                 int want = SSL_get_error(c->ssl, r);
3506                 if(want == SSL_ERROR_ZERO_RETURN) {
3507                         return 0; /* closed */
3508                 } else if(want == SSL_ERROR_WANT_READ) {
3509                         c->ssl_shake_state = comm_ssl_shake_hs_read;
3510                         comm_point_listen_for_rw(c, 1, 0);
3511                         return 1; /* wait for read condition */
3512                 } else if(want == SSL_ERROR_WANT_WRITE) {
3513                         return 1; /* write more later */
3514                 } else if(want == SSL_ERROR_SYSCALL) {
3515 #ifdef EPIPE
3516                         if(errno == EPIPE && verbosity < 2)
3517                                 return 0; /* silence 'broken pipe' */
3518 #endif
3519                         if(errno != 0)
3520                                 log_err("SSL_write syscall: %s",
3521                                         strerror(errno));
3522                         return 0;
3523                 }
3524                 log_crypto_err("could not SSL_write");
3525                 return 0;
3526         }
3527         sldns_buffer_skip(c->buffer, (ssize_t)r);
3528         return 1;
3529 #else
3530         (void)c;
3531         return 0;
3532 #endif /* HAVE_SSL */
3533 }
3534
3535 /** write more data for http */
3536 static int
3537 http_write_more(int fd, struct comm_point* c)
3538 {
3539         ssize_t r;
3540         log_assert(sldns_buffer_remaining(c->buffer) > 0);
3541         r = send(fd, (void*)sldns_buffer_current(c->buffer),
3542                 sldns_buffer_remaining(c->buffer), 0);
3543         if(r == -1) {
3544 #ifndef USE_WINSOCK
3545                 if(errno == EINTR || errno == EAGAIN)
3546                         return 1;
3547 #else
3548                 if(WSAGetLastError() == WSAEINPROGRESS)
3549                         return 1;
3550                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
3551                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
3552                         return 1;
3553                 }
3554 #endif
3555                 log_err_addr("http send r", sock_strerror(errno),
3556                         &c->repinfo.remote_addr, c->repinfo.remote_addrlen);
3557                 return 0;
3558         }
3559         sldns_buffer_skip(c->buffer, r);
3560         return 1;
3561 }
3562
3563 #ifdef HAVE_NGHTTP2
3564 ssize_t http2_send_cb(nghttp2_session* ATTR_UNUSED(session), const uint8_t* buf,
3565         size_t len, int ATTR_UNUSED(flags), void* cb_arg)
3566 {
3567         ssize_t ret;
3568         struct http2_session* h2_session = (struct http2_session*)cb_arg;
3569         log_assert(h2_session->c->type == comm_http);
3570         log_assert(h2_session->c->h2_session);
3571
3572 #ifdef HAVE_SSL
3573         if(h2_session->c->ssl) {
3574                 int r;
3575                 ERR_clear_error();
3576                 r = SSL_write(h2_session->c->ssl, buf, len);
3577                 if(r <= 0) {
3578                         int want = SSL_get_error(h2_session->c->ssl, r);
3579                         if(want == SSL_ERROR_ZERO_RETURN) {
3580                                 return NGHTTP2_ERR_CALLBACK_FAILURE;
3581                         } else if(want == SSL_ERROR_WANT_READ) {
3582                                 h2_session->c->ssl_shake_state = comm_ssl_shake_hs_read;
3583                                 comm_point_listen_for_rw(h2_session->c, 1, 0);
3584                                 return NGHTTP2_ERR_WOULDBLOCK;
3585                         } else if(want == SSL_ERROR_WANT_WRITE) {
3586                                 return NGHTTP2_ERR_WOULDBLOCK;
3587                         } else if(want == SSL_ERROR_SYSCALL) {
3588 #ifdef EPIPE
3589                                 if(errno == EPIPE && verbosity < 2)
3590                                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3591 #endif
3592                                 if(errno != 0)
3593                                         log_err("SSL_write syscall: %s",
3594                                                 strerror(errno));
3595                                 return NGHTTP2_ERR_CALLBACK_FAILURE;
3596                         }
3597                         log_crypto_err("could not SSL_write");
3598                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3599                 }
3600                 return r;
3601         }
3602 #endif /* HAVE_SSL */
3603
3604         ret = send(h2_session->c->fd, buf, len, 0);
3605         if(ret == 0) {
3606                 return NGHTTP2_ERR_CALLBACK_FAILURE;
3607         } else if(ret < 0) {
3608 #ifndef USE_WINSOCK
3609                 if(errno == EINTR || errno == EAGAIN)
3610                         return NGHTTP2_ERR_WOULDBLOCK;
3611 #ifdef EPIPE
3612                 if(errno == EPIPE && verbosity < 2)
3613                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3614 #endif
3615 #ifdef ECONNRESET
3616                 if(errno == ECONNRESET && verbosity < 2)
3617                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3618 #endif
3619                 log_err_addr("could not http2 write: %s", strerror(errno),
3620                         &h2_session->c->repinfo.remote_addr,
3621                         h2_session->c->repinfo.remote_addrlen);
3622 #else /* USE_WINSOCK */
3623                 if(WSAGetLastError() == WSAENOTCONN)
3624                         return NGHTTP2_ERR_WOULDBLOCK;
3625                 if(WSAGetLastError() == WSAEINPROGRESS)
3626                         return NGHTTP2_ERR_WOULDBLOCK;
3627                 if(WSAGetLastError() == WSAEWOULDBLOCK) {
3628                         ub_winsock_tcp_wouldblock(h2_session->c->ev->ev,
3629                                 UB_EV_WRITE);
3630                         return NGHTTP2_ERR_WOULDBLOCK;
3631                 }
3632                 if(WSAGetLastError() == WSAECONNRESET && verbosity < 2)
3633                         return NGHTTP2_ERR_CALLBACK_FAILURE;
3634                 log_err_addr("could not http2 write: %s",
3635                         wsa_strerror(WSAGetLastError()),
3636                         &h2_session->c->repinfo.remote_addr,
3637                         h2_session->c->repinfo.remote_addrlen);
3638 #endif
3639                 return NGHTTP2_ERR_CALLBACK_FAILURE;
3640         }
3641         return ret;
3642 }
3643 #endif /* HAVE_NGHTTP2 */
3644
3645 /** Handle http2 writing */
3646 static int
3647 comm_point_http2_handle_write(int ATTR_UNUSED(fd), struct comm_point* c)
3648 {
3649 #ifdef HAVE_NGHTTP2
3650         int ret;
3651         log_assert(c->h2_session);
3652
3653         ret = nghttp2_session_send(c->h2_session->session);
3654         if(ret) {
3655                 verbose(VERB_QUERY, "http2: session_send failed, "
3656                         "error: %s", nghttp2_strerror(ret));
3657                 return 0;
3658         }
3659
3660         if(nghttp2_session_want_read(c->h2_session->session)) {
3661                 c->tcp_is_reading = 1;
3662                 comm_point_stop_listening(c);
3663                 comm_point_start_listening(c, -1, adjusted_tcp_timeout(c));
3664         } else if(!nghttp2_session_want_write(c->h2_session->session))
3665                 return 0; /* connection can be closed */
3666         return 1;
3667 #else
3668         (void)c;
3669         return 0;
3670 #endif
3671 }
3672
3673 /**
3674  * Handle http writing callback.
3675  * @param fd: file descriptor of socket.
3676  * @param c: comm point to write buffer out of.
3677  * @return: 0 on error
3678  */
3679 static int
3680 comm_point_http_handle_write(int fd, struct comm_point* c)
3681 {
3682         log_assert(c->type == comm_http);
3683         log_assert(fd != -1);
3684
3685         /* check pending connect errors, if that fails, we wait for more,
3686          * or we can continue to write contents */
3687         if(c->tcp_check_nb_connect) {
3688                 int r = http_check_connect(fd, c);
3689                 if(r == 0) return 0;
3690                 if(r == 1) return 1;
3691                 c->tcp_check_nb_connect = 0;
3692         }
3693         /* if we are in ssl handshake, handle SSL handshake */
3694 #ifdef HAVE_SSL
3695         if(c->ssl && c->ssl_shake_state != comm_ssl_shake_none) {
3696                 if(!ssl_handshake(c))
3697                         return 0;
3698                 if(c->ssl_shake_state != comm_ssl_shake_none)
3699                         return 1;
3700         }
3701 #endif /* HAVE_SSL */
3702         if(c->tcp_is_reading)
3703                 return 1;
3704
3705         if(c->use_h2) {
3706                 return comm_point_http2_handle_write(fd, c);
3707         }
3708
3709         /* http version is <= http/1.1 */
3710
3711         if(c->http_min_version >= http_version_2) {
3712                 /* HTTP/2 failed, not allowed to use lower version. */
3713                 return 0;
3714         }
3715
3716         /* if we are writing, write more */
3717         if(c->ssl) {
3718                 if(!ssl_http_write_more(c))
3719                         return 0;
3720         } else {
3721                 if(!http_write_more(fd, c))
3722                         return 0;
3723         }
3724
3725         /* we write a single buffer contents, that can contain
3726          * the http request, and then flip to read the results */
3727         /* see if write is done */
3728         if(sldns_buffer_remaining(c->buffer) == 0) {
3729                 sldns_buffer_clear(c->buffer);
3730                 if(c->tcp_do_toggle_rw)
3731                         c->tcp_is_reading = 1;
3732                 c->tcp_byte_count = 0;
3733                 /* switch from listening(write) to listening(read) */
3734                 comm_point_stop_listening(c);
3735                 comm_point_start_listening(c, -1, -1);
3736         }
3737         return 1;
3738 }
3739
3740 void
3741 comm_point_http_handle_callback(int fd, short event, void* arg)
3742 {
3743         struct comm_point* c = (struct comm_point*)arg;
3744         log_assert(c->type == comm_http);
3745         ub_comm_base_now(c->ev->base);
3746
3747         if(event&UB_EV_TIMEOUT) {
3748                 verbose(VERB_QUERY, "http took too long, dropped");
3749                 reclaim_http_handler(c);
3750                 if(!c->tcp_do_close) {
3751                         fptr_ok(fptr_whitelist_comm_point(c->callback));
3752                         (void)(*c->callback)(c, c->cb_arg,
3753                                 NETEVENT_TIMEOUT, NULL);
3754                 }
3755                 return;
3756         }
3757         if(event&UB_EV_READ) {
3758                 if(!comm_point_http_handle_read(fd, c)) {
3759                         reclaim_http_handler(c);
3760                         if(!c->tcp_do_close) {
3761                                 fptr_ok(fptr_whitelist_comm_point(
3762                                         c->callback));
3763                                 (void)(*c->callback)(c, c->cb_arg,
3764                                         NETEVENT_CLOSED, NULL);
3765                         }
3766                 }
3767                 return;
3768         }
3769         if(event&UB_EV_WRITE) {
3770                 if(!comm_point_http_handle_write(fd, c)) {
3771                         reclaim_http_handler(c);
3772                         if(!c->tcp_do_close) {
3773                                 fptr_ok(fptr_whitelist_comm_point(
3774                                         c->callback));
3775                                 (void)(*c->callback)(c, c->cb_arg,
3776                                         NETEVENT_CLOSED, NULL);
3777                         }
3778                 }
3779                 return;
3780         }
3781         log_err("Ignored event %d for httphdl.", event);
3782 }
3783
3784 void comm_point_local_handle_callback(int fd, short event, void* arg)
3785 {
3786         struct comm_point* c = (struct comm_point*)arg;
3787         log_assert(c->type == comm_local);
3788         ub_comm_base_now(c->ev->base);
3789
3790         if(event&UB_EV_READ) {
3791                 if(!comm_point_tcp_handle_read(fd, c, 1)) {
3792                         fptr_ok(fptr_whitelist_comm_point(c->callback));
3793                         (void)(*c->callback)(c, c->cb_arg, NETEVENT_CLOSED,
3794                                 NULL);
3795                 }
3796                 return;
3797         }
3798         log_err("Ignored event %d for localhdl.", event);
3799 }
3800
3801 void comm_point_raw_handle_callback(int ATTR_UNUSED(fd),
3802         short event, void* arg)
3803 {
3804         struct comm_point* c = (struct comm_point*)arg;
3805         int err = NETEVENT_NOERROR;
3806         log_assert(c->type == comm_raw);
3807         ub_comm_base_now(c->ev->base);
3808
3809         if(event&UB_EV_TIMEOUT)
3810                 err = NETEVENT_TIMEOUT;
3811         fptr_ok(fptr_whitelist_comm_point_raw(c->callback));
3812         (void)(*c->callback)(c, c->cb_arg, err, NULL);
3813 }
3814
3815 struct comm_point*
3816 comm_point_create_udp(struct comm_base *base, int fd, sldns_buffer* buffer,
3817         int pp2_enabled, comm_point_callback_type* callback,
3818         void* callback_arg, struct unbound_socket* socket)
3819 {
3820         struct comm_point* c = (struct comm_point*)calloc(1,
3821                 sizeof(struct comm_point));
3822         short evbits;
3823         if(!c)
3824                 return NULL;
3825         c->ev = (struct internal_event*)calloc(1,
3826                 sizeof(struct internal_event));
3827         if(!c->ev) {
3828                 free(c);
3829                 return NULL;
3830         }
3831         c->ev->base = base;
3832         c->fd = fd;
3833         c->buffer = buffer;
3834         c->timeout = NULL;
3835         c->tcp_is_reading = 0;
3836         c->tcp_byte_count = 0;
3837         c->tcp_parent = NULL;
3838         c->max_tcp_count = 0;
3839         c->cur_tcp_count = 0;
3840         c->tcp_handlers = NULL;
3841         c->tcp_free = NULL;
3842         c->type = comm_udp;
3843         c->tcp_do_close = 0;
3844         c->do_not_close = 0;
3845         c->tcp_do_toggle_rw = 0;
3846         c->tcp_check_nb_connect = 0;
3847 #ifdef USE_MSG_FASTOPEN
3848         c->tcp_do_fastopen = 0;
3849 #endif
3850 #ifdef USE_DNSCRYPT
3851         c->dnscrypt = 0;
3852         c->dnscrypt_buffer = buffer;
3853 #endif
3854         c->inuse = 0;
3855         c->callback = callback;
3856         c->cb_arg = callback_arg;
3857         c->socket = socket;
3858         c->pp2_enabled = pp2_enabled;
3859         c->pp2_header_state = pp2_header_none;
3860         evbits = UB_EV_READ | UB_EV_PERSIST;
3861         /* ub_event stuff */
3862         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3863 #ifdef USE_WINSOCK
3864                 comm_point_udp_callback, c);
3865 #else
3866                 comm_point_udp_ancil_callback, c);
3867 #endif
3868         if(c->ev->ev == NULL) {
3869                 log_err("could not baseset udp event");
3870                 comm_point_delete(c);
3871                 return NULL;
3872         }
3873         if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
3874                 log_err("could not add udp event");
3875                 comm_point_delete(c);
3876                 return NULL;
3877         }
3878         c->event_added = 1;
3879         return c;
3880 }
3881
3882 struct comm_point*
3883 comm_point_create_udp_ancil(struct comm_base *base, int fd,
3884         sldns_buffer* buffer, int pp2_enabled,
3885         comm_point_callback_type* callback, void* callback_arg, struct unbound_socket* socket)
3886 {
3887         struct comm_point* c = (struct comm_point*)calloc(1,
3888                 sizeof(struct comm_point));
3889         short evbits;
3890         if(!c)
3891                 return NULL;
3892         c->ev = (struct internal_event*)calloc(1,
3893                 sizeof(struct internal_event));
3894         if(!c->ev) {
3895                 free(c);
3896                 return NULL;
3897         }
3898         c->ev->base = base;
3899         c->fd = fd;
3900         c->buffer = buffer;
3901         c->timeout = NULL;
3902         c->tcp_is_reading = 0;
3903         c->tcp_byte_count = 0;
3904         c->tcp_parent = NULL;
3905         c->max_tcp_count = 0;
3906         c->cur_tcp_count = 0;
3907         c->tcp_handlers = NULL;
3908         c->tcp_free = NULL;
3909         c->type = comm_udp;
3910         c->tcp_do_close = 0;
3911         c->do_not_close = 0;
3912 #ifdef USE_DNSCRYPT
3913         c->dnscrypt = 0;
3914         c->dnscrypt_buffer = buffer;
3915 #endif
3916         c->inuse = 0;
3917         c->tcp_do_toggle_rw = 0;
3918         c->tcp_check_nb_connect = 0;
3919 #ifdef USE_MSG_FASTOPEN
3920         c->tcp_do_fastopen = 0;
3921 #endif
3922         c->callback = callback;
3923         c->cb_arg = callback_arg;
3924         c->socket = socket;
3925         c->pp2_enabled = pp2_enabled;
3926         c->pp2_header_state = pp2_header_none;
3927         evbits = UB_EV_READ | UB_EV_PERSIST;
3928         /* ub_event stuff */
3929         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
3930                 comm_point_udp_ancil_callback, c);
3931         if(c->ev->ev == NULL) {
3932                 log_err("could not baseset udp event");
3933                 comm_point_delete(c);
3934                 return NULL;
3935         }
3936         if(fd!=-1 && ub_event_add(c->ev->ev, c->timeout) != 0 ) {
3937                 log_err("could not add udp event");
3938                 comm_point_delete(c);
3939                 return NULL;
3940         }
3941         c->event_added = 1;
3942         return c;
3943 }
3944
3945 static struct comm_point*
3946 comm_point_create_tcp_handler(struct comm_base *base,
3947         struct comm_point* parent, size_t bufsize,
3948         struct sldns_buffer* spoolbuf, comm_point_callback_type* callback,
3949         void* callback_arg, struct unbound_socket* socket)
3950 {
3951         struct comm_point* c = (struct comm_point*)calloc(1,
3952                 sizeof(struct comm_point));
3953         short evbits;
3954         if(!c)
3955                 return NULL;
3956         c->ev = (struct internal_event*)calloc(1,
3957                 sizeof(struct internal_event));
3958         if(!c->ev) {
3959                 free(c);
3960                 return NULL;
3961         }
3962         c->ev->base = base;
3963         c->fd = -1;
3964         c->buffer = sldns_buffer_new(bufsize);
3965         if(!c->buffer) {
3966                 free(c->ev);
3967                 free(c);
3968                 return NULL;
3969         }
3970         c->timeout = (struct timeval*)malloc(sizeof(struct timeval));
3971         if(!c->timeout) {
3972                 sldns_buffer_free(c->buffer);
3973                 free(c->ev);
3974                 free(c);
3975                 return NULL;
3976         }
3977         c->tcp_is_reading = 0;
3978         c->tcp_byte_count = 0;
3979         c->tcp_parent = parent;
3980         c->tcp_timeout_msec = parent->tcp_timeout_msec;
3981         c->tcp_conn_limit = parent->tcp_conn_limit;
3982         c->tcl_addr = NULL;
3983         c->tcp_keepalive = 0;
3984         c->max_tcp_count = 0;
3985         c->cur_tcp_count = 0;
3986         c->tcp_handlers = NULL;
3987         c->tcp_free = NULL;
3988         c->type = comm_tcp;
3989         c->tcp_do_close = 0;
3990         c->do_not_close = 0;
3991         c->tcp_do_toggle_rw = 1;
3992         c->tcp_check_nb_connect = 0;
3993 #ifdef USE_MSG_FASTOPEN
3994         c->tcp_do_fastopen = 0;
3995 #endif
3996 #ifdef USE_DNSCRYPT
3997         c->dnscrypt = 0;
3998         /* We don't know just yet if this is a dnscrypt channel. Allocation
3999          * will be done when handling the callback. */
4000         c->dnscrypt_buffer = c->buffer;
4001 #endif
4002         c->repinfo.c = c;
4003         c->callback = callback;
4004         c->cb_arg = callback_arg;
4005         c->socket = socket;
4006         c->pp2_enabled = parent->pp2_enabled;
4007         c->pp2_header_state = pp2_header_none;
4008         if(spoolbuf) {
4009                 c->tcp_req_info = tcp_req_info_create(spoolbuf);
4010                 if(!c->tcp_req_info) {
4011                         log_err("could not create tcp commpoint");
4012                         sldns_buffer_free(c->buffer);
4013                         free(c->timeout);
4014                         free(c->ev);
4015                         free(c);
4016                         return NULL;
4017                 }
4018                 c->tcp_req_info->cp = c;
4019                 c->tcp_do_close = 1;
4020                 c->tcp_do_toggle_rw = 0;
4021         }
4022         /* add to parent free list */
4023         c->tcp_free = parent->tcp_free;
4024         parent->tcp_free = c;
4025         /* ub_event stuff */
4026         evbits = UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT;
4027         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4028                 comm_point_tcp_handle_callback, c);
4029         if(c->ev->ev == NULL)
4030         {
4031                 log_err("could not basetset tcphdl event");
4032                 parent->tcp_free = c->tcp_free;
4033                 tcp_req_info_delete(c->tcp_req_info);
4034                 sldns_buffer_free(c->buffer);
4035                 free(c->timeout);
4036                 free(c->ev);
4037                 free(c);
4038                 return NULL;
4039         }
4040         return c;
4041 }
4042
4043 static struct comm_point*
4044 comm_point_create_http_handler(struct comm_base *base,
4045         struct comm_point* parent, size_t bufsize, int harden_large_queries,
4046         uint32_t http_max_streams, char* http_endpoint,
4047         comm_point_callback_type* callback, void* callback_arg,
4048         struct unbound_socket* socket)
4049 {
4050         struct comm_point* c = (struct comm_point*)calloc(1,
4051                 sizeof(struct comm_point));
4052         short evbits;
4053         if(!c)
4054                 return NULL;
4055         c->ev = (struct internal_event*)calloc(1,
4056                 sizeof(struct internal_event));
4057         if(!c->ev) {
4058                 free(c);
4059                 return NULL;
4060         }
4061         c->ev->base = base;
4062         c->fd = -1;
4063         c->buffer = sldns_buffer_new(bufsize);
4064         if(!c->buffer) {
4065                 free(c->ev);
4066                 free(c);
4067                 return NULL;
4068         }
4069         c->timeout = (struct timeval*)malloc(sizeof(struct timeval));
4070         if(!c->timeout) {
4071                 sldns_buffer_free(c->buffer);
4072                 free(c->ev);
4073                 free(c);
4074                 return NULL;
4075         }
4076         c->tcp_is_reading = 0;
4077         c->tcp_byte_count = 0;
4078         c->tcp_parent = parent;
4079         c->tcp_timeout_msec = parent->tcp_timeout_msec;
4080         c->tcp_conn_limit = parent->tcp_conn_limit;
4081         c->tcl_addr = NULL;
4082         c->tcp_keepalive = 0;
4083         c->max_tcp_count = 0;
4084         c->cur_tcp_count = 0;
4085         c->tcp_handlers = NULL;
4086         c->tcp_free = NULL;
4087         c->type = comm_http;
4088         c->tcp_do_close = 1;
4089         c->do_not_close = 0;
4090         c->tcp_do_toggle_rw = 1; /* will be set to 0 after http2 upgrade */
4091         c->tcp_check_nb_connect = 0;
4092 #ifdef USE_MSG_FASTOPEN
4093         c->tcp_do_fastopen = 0;
4094 #endif
4095 #ifdef USE_DNSCRYPT
4096         c->dnscrypt = 0;
4097         c->dnscrypt_buffer = NULL;
4098 #endif
4099         c->repinfo.c = c;
4100         c->callback = callback;
4101         c->cb_arg = callback_arg;
4102         c->socket = socket;
4103         c->pp2_enabled = 0;
4104         c->pp2_header_state = pp2_header_none;
4105
4106         c->http_min_version = http_version_2;
4107         c->http2_stream_max_qbuffer_size = bufsize;
4108         if(harden_large_queries && bufsize > 512)
4109                 c->http2_stream_max_qbuffer_size = 512;
4110         c->http2_max_streams = http_max_streams;
4111         if(!(c->http_endpoint = strdup(http_endpoint))) {
4112                 log_err("could not strdup http_endpoint");
4113                 sldns_buffer_free(c->buffer);
4114                 free(c->timeout);
4115                 free(c->ev);
4116                 free(c);
4117                 return NULL;
4118         }
4119         c->use_h2 = 0;
4120 #ifdef HAVE_NGHTTP2
4121         if(!(c->h2_session = http2_session_create(c))) {
4122                 log_err("could not create http2 session");
4123                 free(c->http_endpoint);
4124                 sldns_buffer_free(c->buffer);
4125                 free(c->timeout);
4126                 free(c->ev);
4127                 free(c);
4128                 return NULL;
4129         }
4130         if(!(c->h2_session->callbacks = http2_req_callbacks_create())) {
4131                 log_err("could not create http2 callbacks");
4132                 http2_session_delete(c->h2_session);
4133                 free(c->http_endpoint);
4134                 sldns_buffer_free(c->buffer);
4135                 free(c->timeout);
4136                 free(c->ev);
4137                 free(c);
4138                 return NULL;
4139         }
4140 #endif
4141
4142         /* add to parent free list */
4143         c->tcp_free = parent->tcp_free;
4144         parent->tcp_free = c;
4145         /* ub_event stuff */
4146         evbits = UB_EV_PERSIST | UB_EV_READ | UB_EV_TIMEOUT;
4147         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4148                 comm_point_http_handle_callback, c);
4149         if(c->ev->ev == NULL)
4150         {
4151                 log_err("could not set http handler event");
4152                 parent->tcp_free = c->tcp_free;
4153                 http2_session_delete(c->h2_session);
4154                 sldns_buffer_free(c->buffer);
4155                 free(c->timeout);
4156                 free(c->ev);
4157                 free(c);
4158                 return NULL;
4159         }
4160         return c;
4161 }
4162
4163 struct comm_point*
4164 comm_point_create_tcp(struct comm_base *base, int fd, int num,
4165         int idle_timeout, int harden_large_queries,
4166         uint32_t http_max_streams, char* http_endpoint,
4167         struct tcl_list* tcp_conn_limit, size_t bufsize,
4168         struct sldns_buffer* spoolbuf, enum listen_type port_type,
4169         int pp2_enabled, comm_point_callback_type* callback,
4170         void* callback_arg, struct unbound_socket* socket)
4171 {
4172         struct comm_point* c = (struct comm_point*)calloc(1,
4173                 sizeof(struct comm_point));
4174         short evbits;
4175         int i;
4176         /* first allocate the TCP accept listener */
4177         if(!c)
4178                 return NULL;
4179         c->ev = (struct internal_event*)calloc(1,
4180                 sizeof(struct internal_event));
4181         if(!c->ev) {
4182                 free(c);
4183                 return NULL;
4184         }
4185         c->ev->base = base;
4186         c->fd = fd;
4187         c->buffer = NULL;
4188         c->timeout = NULL;
4189         c->tcp_is_reading = 0;
4190         c->tcp_byte_count = 0;
4191         c->tcp_timeout_msec = idle_timeout;
4192         c->tcp_conn_limit = tcp_conn_limit;
4193         c->tcl_addr = NULL;
4194         c->tcp_keepalive = 0;
4195         c->tcp_parent = NULL;
4196         c->max_tcp_count = num;
4197         c->cur_tcp_count = 0;
4198         c->tcp_handlers = (struct comm_point**)calloc((size_t)num,
4199                 sizeof(struct comm_point*));
4200         if(!c->tcp_handlers) {
4201                 free(c->ev);
4202                 free(c);
4203                 return NULL;
4204         }
4205         c->tcp_free = NULL;
4206         c->type = comm_tcp_accept;
4207         c->tcp_do_close = 0;
4208         c->do_not_close = 0;
4209         c->tcp_do_toggle_rw = 0;
4210         c->tcp_check_nb_connect = 0;
4211 #ifdef USE_MSG_FASTOPEN
4212         c->tcp_do_fastopen = 0;
4213 #endif
4214 #ifdef USE_DNSCRYPT
4215         c->dnscrypt = 0;
4216         c->dnscrypt_buffer = NULL;
4217 #endif
4218         c->callback = NULL;
4219         c->cb_arg = NULL;
4220         c->socket = socket;
4221         c->pp2_enabled = (port_type==listen_type_http?0:pp2_enabled);
4222         c->pp2_header_state = pp2_header_none;
4223         evbits = UB_EV_READ | UB_EV_PERSIST;
4224         /* ub_event stuff */
4225         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4226                 comm_point_tcp_accept_callback, c);
4227         if(c->ev->ev == NULL) {
4228                 log_err("could not baseset tcpacc event");
4229                 comm_point_delete(c);
4230                 return NULL;
4231         }
4232         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
4233                 log_err("could not add tcpacc event");
4234                 comm_point_delete(c);
4235                 return NULL;
4236         }
4237         c->event_added = 1;
4238         /* now prealloc the handlers */
4239         for(i=0; i<num; i++) {
4240                 if(port_type == listen_type_tcp ||
4241                         port_type == listen_type_ssl ||
4242                         port_type == listen_type_tcp_dnscrypt) {
4243                         c->tcp_handlers[i] = comm_point_create_tcp_handler(base,
4244                                 c, bufsize, spoolbuf, callback, callback_arg, socket);
4245                 } else if(port_type == listen_type_http) {
4246                         c->tcp_handlers[i] = comm_point_create_http_handler(
4247                                 base, c, bufsize, harden_large_queries,
4248                                 http_max_streams, http_endpoint,
4249                                 callback, callback_arg, socket);
4250                 }
4251                 else {
4252                         log_err("could not create tcp handler, unknown listen "
4253                                 "type");
4254                         return NULL;
4255                 }
4256                 if(!c->tcp_handlers[i]) {
4257                         comm_point_delete(c);
4258                         return NULL;
4259                 }
4260         }
4261
4262         return c;
4263 }
4264
4265 struct comm_point*
4266 comm_point_create_tcp_out(struct comm_base *base, size_t bufsize,
4267         comm_point_callback_type* callback, void* callback_arg)
4268 {
4269         struct comm_point* c = (struct comm_point*)calloc(1,
4270                 sizeof(struct comm_point));
4271         short evbits;
4272         if(!c)
4273                 return NULL;
4274         c->ev = (struct internal_event*)calloc(1,
4275                 sizeof(struct internal_event));
4276         if(!c->ev) {
4277                 free(c);
4278                 return NULL;
4279         }
4280         c->ev->base = base;
4281         c->fd = -1;
4282         c->buffer = sldns_buffer_new(bufsize);
4283         if(!c->buffer) {
4284                 free(c->ev);
4285                 free(c);
4286                 return NULL;
4287         }
4288         c->timeout = NULL;
4289         c->tcp_is_reading = 0;
4290         c->tcp_byte_count = 0;
4291         c->tcp_timeout_msec = TCP_QUERY_TIMEOUT;
4292         c->tcp_conn_limit = NULL;
4293         c->tcl_addr = NULL;
4294         c->tcp_keepalive = 0;
4295         c->tcp_parent = NULL;
4296         c->max_tcp_count = 0;
4297         c->cur_tcp_count = 0;
4298         c->tcp_handlers = NULL;
4299         c->tcp_free = NULL;
4300         c->type = comm_tcp;
4301         c->tcp_do_close = 0;
4302         c->do_not_close = 0;
4303         c->tcp_do_toggle_rw = 1;
4304         c->tcp_check_nb_connect = 1;
4305 #ifdef USE_MSG_FASTOPEN
4306         c->tcp_do_fastopen = 1;
4307 #endif
4308 #ifdef USE_DNSCRYPT
4309         c->dnscrypt = 0;
4310         c->dnscrypt_buffer = c->buffer;
4311 #endif
4312         c->repinfo.c = c;
4313         c->callback = callback;
4314         c->cb_arg = callback_arg;
4315         c->pp2_enabled = 0;
4316         c->pp2_header_state = pp2_header_none;
4317         evbits = UB_EV_PERSIST | UB_EV_WRITE;
4318         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4319                 comm_point_tcp_handle_callback, c);
4320         if(c->ev->ev == NULL)
4321         {
4322                 log_err("could not baseset tcpout event");
4323                 sldns_buffer_free(c->buffer);
4324                 free(c->ev);
4325                 free(c);
4326                 return NULL;
4327         }
4328
4329         return c;
4330 }
4331
4332 struct comm_point*
4333 comm_point_create_http_out(struct comm_base *base, size_t bufsize,
4334         comm_point_callback_type* callback, void* callback_arg,
4335         sldns_buffer* temp)
4336 {
4337         struct comm_point* c = (struct comm_point*)calloc(1,
4338                 sizeof(struct comm_point));
4339         short evbits;
4340         if(!c)
4341                 return NULL;
4342         c->ev = (struct internal_event*)calloc(1,
4343                 sizeof(struct internal_event));
4344         if(!c->ev) {
4345                 free(c);
4346                 return NULL;
4347         }
4348         c->ev->base = base;
4349         c->fd = -1;
4350         c->buffer = sldns_buffer_new(bufsize);
4351         if(!c->buffer) {
4352                 free(c->ev);
4353                 free(c);
4354                 return NULL;
4355         }
4356         c->timeout = NULL;
4357         c->tcp_is_reading = 0;
4358         c->tcp_byte_count = 0;
4359         c->tcp_parent = NULL;
4360         c->max_tcp_count = 0;
4361         c->cur_tcp_count = 0;
4362         c->tcp_handlers = NULL;
4363         c->tcp_free = NULL;
4364         c->type = comm_http;
4365         c->tcp_do_close = 0;
4366         c->do_not_close = 0;
4367         c->tcp_do_toggle_rw = 1;
4368         c->tcp_check_nb_connect = 1;
4369         c->http_in_headers = 1;
4370         c->http_in_chunk_headers = 0;
4371         c->http_is_chunked = 0;
4372         c->http_temp = temp;
4373 #ifdef USE_MSG_FASTOPEN
4374         c->tcp_do_fastopen = 1;
4375 #endif
4376 #ifdef USE_DNSCRYPT
4377         c->dnscrypt = 0;
4378         c->dnscrypt_buffer = c->buffer;
4379 #endif
4380         c->repinfo.c = c;
4381         c->callback = callback;
4382         c->cb_arg = callback_arg;
4383         c->pp2_enabled = 0;
4384         c->pp2_header_state = pp2_header_none;
4385         evbits = UB_EV_PERSIST | UB_EV_WRITE;
4386         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4387                 comm_point_http_handle_callback, c);
4388         if(c->ev->ev == NULL)
4389         {
4390                 log_err("could not baseset tcpout event");
4391 #ifdef HAVE_SSL
4392                 SSL_free(c->ssl);
4393 #endif
4394                 sldns_buffer_free(c->buffer);
4395                 free(c->ev);
4396                 free(c);
4397                 return NULL;
4398         }
4399
4400         return c;
4401 }
4402
4403 struct comm_point*
4404 comm_point_create_local(struct comm_base *base, int fd, size_t bufsize,
4405         comm_point_callback_type* callback, void* callback_arg)
4406 {
4407         struct comm_point* c = (struct comm_point*)calloc(1,
4408                 sizeof(struct comm_point));
4409         short evbits;
4410         if(!c)
4411                 return NULL;
4412         c->ev = (struct internal_event*)calloc(1,
4413                 sizeof(struct internal_event));
4414         if(!c->ev) {
4415                 free(c);
4416                 return NULL;
4417         }
4418         c->ev->base = base;
4419         c->fd = fd;
4420         c->buffer = sldns_buffer_new(bufsize);
4421         if(!c->buffer) {
4422                 free(c->ev);
4423                 free(c);
4424                 return NULL;
4425         }
4426         c->timeout = NULL;
4427         c->tcp_is_reading = 1;
4428         c->tcp_byte_count = 0;
4429         c->tcp_parent = NULL;
4430         c->max_tcp_count = 0;
4431         c->cur_tcp_count = 0;
4432         c->tcp_handlers = NULL;
4433         c->tcp_free = NULL;
4434         c->type = comm_local;
4435         c->tcp_do_close = 0;
4436         c->do_not_close = 1;
4437         c->tcp_do_toggle_rw = 0;
4438         c->tcp_check_nb_connect = 0;
4439 #ifdef USE_MSG_FASTOPEN
4440         c->tcp_do_fastopen = 0;
4441 #endif
4442 #ifdef USE_DNSCRYPT
4443         c->dnscrypt = 0;
4444         c->dnscrypt_buffer = c->buffer;
4445 #endif
4446         c->callback = callback;
4447         c->cb_arg = callback_arg;
4448         c->pp2_enabled = 0;
4449         c->pp2_header_state = pp2_header_none;
4450         /* ub_event stuff */
4451         evbits = UB_EV_PERSIST | UB_EV_READ;
4452         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4453                 comm_point_local_handle_callback, c);
4454         if(c->ev->ev == NULL) {
4455                 log_err("could not baseset localhdl event");
4456                 free(c->ev);
4457                 free(c);
4458                 return NULL;
4459         }
4460         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
4461                 log_err("could not add localhdl event");
4462                 ub_event_free(c->ev->ev);
4463                 free(c->ev);
4464                 free(c);
4465                 return NULL;
4466         }
4467         c->event_added = 1;
4468         return c;
4469 }
4470
4471 struct comm_point*
4472 comm_point_create_raw(struct comm_base* base, int fd, int writing,
4473         comm_point_callback_type* callback, void* callback_arg)
4474 {
4475         struct comm_point* c = (struct comm_point*)calloc(1,
4476                 sizeof(struct comm_point));
4477         short evbits;
4478         if(!c)
4479                 return NULL;
4480         c->ev = (struct internal_event*)calloc(1,
4481                 sizeof(struct internal_event));
4482         if(!c->ev) {
4483                 free(c);
4484                 return NULL;
4485         }
4486         c->ev->base = base;
4487         c->fd = fd;
4488         c->buffer = NULL;
4489         c->timeout = NULL;
4490         c->tcp_is_reading = 0;
4491         c->tcp_byte_count = 0;
4492         c->tcp_parent = NULL;
4493         c->max_tcp_count = 0;
4494         c->cur_tcp_count = 0;
4495         c->tcp_handlers = NULL;
4496         c->tcp_free = NULL;
4497         c->type = comm_raw;
4498         c->tcp_do_close = 0;
4499         c->do_not_close = 1;
4500         c->tcp_do_toggle_rw = 0;
4501         c->tcp_check_nb_connect = 0;
4502 #ifdef USE_MSG_FASTOPEN
4503         c->tcp_do_fastopen = 0;
4504 #endif
4505 #ifdef USE_DNSCRYPT
4506         c->dnscrypt = 0;
4507         c->dnscrypt_buffer = c->buffer;
4508 #endif
4509         c->callback = callback;
4510         c->cb_arg = callback_arg;
4511         c->pp2_enabled = 0;
4512         c->pp2_header_state = pp2_header_none;
4513         /* ub_event stuff */
4514         if(writing)
4515                 evbits = UB_EV_PERSIST | UB_EV_WRITE;
4516         else    evbits = UB_EV_PERSIST | UB_EV_READ;
4517         c->ev->ev = ub_event_new(base->eb->base, c->fd, evbits,
4518                 comm_point_raw_handle_callback, c);
4519         if(c->ev->ev == NULL) {
4520                 log_err("could not baseset rawhdl event");
4521                 free(c->ev);
4522                 free(c);
4523                 return NULL;
4524         }
4525         if (ub_event_add(c->ev->ev, c->timeout) != 0) {
4526                 log_err("could not add rawhdl event");
4527                 ub_event_free(c->ev->ev);
4528                 free(c->ev);
4529                 free(c);
4530                 return NULL;
4531         }
4532         c->event_added = 1;
4533         return c;
4534 }
4535
4536 void
4537 comm_point_close(struct comm_point* c)
4538 {
4539         if(!c)
4540                 return;
4541         if(c->fd != -1) {
4542                 verbose(5, "comm_point_close of %d: event_del", c->fd);
4543                 if(c->event_added) {
4544                         if(ub_event_del(c->ev->ev) != 0) {
4545                                 log_err("could not event_del on close");
4546                         }
4547                         c->event_added = 0;
4548                 }
4549         }
4550         tcl_close_connection(c->tcl_addr);
4551         if(c->tcp_req_info)
4552                 tcp_req_info_clear(c->tcp_req_info);
4553         if(c->h2_session)
4554                 http2_session_server_delete(c->h2_session);
4555         /* stop the comm point from reading or writing after it is closed. */
4556         if(c->tcp_more_read_again && *c->tcp_more_read_again)
4557                 *c->tcp_more_read_again = 0;
4558         if(c->tcp_more_write_again && *c->tcp_more_write_again)
4559                 *c->tcp_more_write_again = 0;
4560
4561         /* close fd after removing from event lists, or epoll.. is messed up */
4562         if(c->fd != -1 && !c->do_not_close) {
4563 #ifdef USE_WINSOCK
4564                 if(c->type == comm_tcp || c->type == comm_http) {
4565                         /* delete sticky events for the fd, it gets closed */
4566                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_READ);
4567                         ub_winsock_tcp_wouldblock(c->ev->ev, UB_EV_WRITE);
4568                 }
4569 #endif
4570                 verbose(VERB_ALGO, "close fd %d", c->fd);
4571                 sock_close(c->fd);
4572         }
4573         c->fd = -1;
4574 }
4575
4576 void
4577 comm_point_delete(struct comm_point* c)
4578 {
4579         if(!c)
4580                 return;
4581         if((c->type == comm_tcp || c->type == comm_http) && c->ssl) {
4582 #ifdef HAVE_SSL
4583                 SSL_shutdown(c->ssl);
4584                 SSL_free(c->ssl);
4585 #endif
4586         }
4587         if(c->type == comm_http && c->http_endpoint) {
4588                 free(c->http_endpoint);
4589                 c->http_endpoint = NULL;
4590         }
4591         comm_point_close(c);
4592         if(c->tcp_handlers) {
4593                 int i;
4594                 for(i=0; i<c->max_tcp_count; i++)
4595                         comm_point_delete(c->tcp_handlers[i]);
4596                 free(c->tcp_handlers);
4597         }
4598         free(c->timeout);
4599         if(c->type == comm_tcp || c->type == comm_local || c->type == comm_http) {
4600                 sldns_buffer_free(c->buffer);
4601 #ifdef USE_DNSCRYPT
4602                 if(c->dnscrypt && c->dnscrypt_buffer != c->buffer) {
4603                         sldns_buffer_free(c->dnscrypt_buffer);
4604                 }
4605 #endif
4606                 if(c->tcp_req_info) {
4607                         tcp_req_info_delete(c->tcp_req_info);
4608                 }
4609                 if(c->h2_session) {
4610                         http2_session_delete(c->h2_session);
4611                 }
4612         }
4613         ub_event_free(c->ev->ev);
4614         free(c->ev);
4615         free(c);
4616 }
4617
4618 void
4619 comm_point_send_reply(struct comm_reply *repinfo)
4620 {
4621         struct sldns_buffer* buffer;
4622         log_assert(repinfo && repinfo->c);
4623 #ifdef USE_DNSCRYPT
4624         buffer = repinfo->c->dnscrypt_buffer;
4625         if(!dnsc_handle_uncurved_request(repinfo)) {
4626                 return;
4627         }
4628 #else
4629         buffer = repinfo->c->buffer;
4630 #endif
4631         if(repinfo->c->type == comm_udp) {
4632                 if(repinfo->srctype)
4633                         comm_point_send_udp_msg_if(repinfo->c, buffer,
4634                                 (struct sockaddr*)&repinfo->remote_addr,
4635                                 repinfo->remote_addrlen, repinfo);
4636                 else
4637                         comm_point_send_udp_msg(repinfo->c, buffer,
4638                                 (struct sockaddr*)&repinfo->remote_addr,
4639                                 repinfo->remote_addrlen, 0);
4640 #ifdef USE_DNSTAP
4641                 /*
4642                  * sending src (client)/dst (local service) addresses over DNSTAP from udp callback
4643                  */
4644                 if(repinfo->c->dtenv != NULL && repinfo->c->dtenv->log_client_response_messages) {
4645                         log_addr(VERB_ALGO, "from local addr", (void*)repinfo->c->socket->addr->ai_addr, repinfo->c->socket->addr->ai_addrlen);
4646                         log_addr(VERB_ALGO, "response to client", &repinfo->client_addr, repinfo->client_addrlen);
4647                         dt_msg_send_client_response(repinfo->c->dtenv, &repinfo->client_addr, (void*)repinfo->c->socket->addr->ai_addr, repinfo->c->type, repinfo->c->buffer);
4648                 }
4649 #endif
4650         } else {
4651 #ifdef USE_DNSTAP
4652                 /*
4653                  * sending src (client)/dst (local service) addresses over DNSTAP from TCP callback
4654                  */
4655                 if(repinfo->c->tcp_parent->dtenv != NULL && repinfo->c->tcp_parent->dtenv->log_client_response_messages) {
4656                         log_addr(VERB_ALGO, "from local addr", (void*)repinfo->c->socket->addr->ai_addr, repinfo->c->socket->addr->ai_addrlen);
4657                         log_addr(VERB_ALGO, "response to client", &repinfo->client_addr, repinfo->client_addrlen);
4658                         dt_msg_send_client_response(repinfo->c->tcp_parent->dtenv, &repinfo->client_addr, (void*)repinfo->c->socket->addr->ai_addr, repinfo->c->type,
4659                                 ( repinfo->c->tcp_req_info? repinfo->c->tcp_req_info->spool_buffer: repinfo->c->buffer ));
4660                 }
4661 #endif
4662                 if(repinfo->c->tcp_req_info) {
4663                         tcp_req_info_send_reply(repinfo->c->tcp_req_info);
4664                 } else if(repinfo->c->use_h2) {
4665                         if(!http2_submit_dns_response(repinfo->c->h2_session)) {
4666                                 comm_point_drop_reply(repinfo);
4667                                 return;
4668                         }
4669                         repinfo->c->h2_stream = NULL;
4670                         repinfo->c->tcp_is_reading = 0;
4671                         comm_point_stop_listening(repinfo->c);
4672                         comm_point_start_listening(repinfo->c, -1,
4673                                 adjusted_tcp_timeout(repinfo->c));
4674                         return;
4675                 } else {
4676                         comm_point_start_listening(repinfo->c, -1,
4677                                 adjusted_tcp_timeout(repinfo->c));
4678                 }
4679         }
4680 }
4681
4682 void
4683 comm_point_drop_reply(struct comm_reply* repinfo)
4684 {
4685         if(!repinfo)
4686                 return;
4687         log_assert(repinfo->c);
4688         log_assert(repinfo->c->type != comm_tcp_accept);
4689         if(repinfo->c->type == comm_udp)
4690                 return;
4691         if(repinfo->c->tcp_req_info)
4692                 repinfo->c->tcp_req_info->is_drop = 1;
4693         if(repinfo->c->type == comm_http) {
4694                 if(repinfo->c->h2_session) {
4695                         repinfo->c->h2_session->is_drop = 1;
4696                         if(!repinfo->c->h2_session->postpone_drop)
4697                                 reclaim_http_handler(repinfo->c);
4698                         return;
4699                 }
4700                 reclaim_http_handler(repinfo->c);
4701                 return;
4702         }
4703         reclaim_tcp_handler(repinfo->c);
4704 }
4705
4706 void
4707 comm_point_stop_listening(struct comm_point* c)
4708 {
4709         verbose(VERB_ALGO, "comm point stop listening %d", c->fd);
4710         if(c->event_added) {
4711                 if(ub_event_del(c->ev->ev) != 0) {
4712                         log_err("event_del error to stoplisten");
4713                 }
4714                 c->event_added = 0;
4715         }
4716 }
4717
4718 void
4719 comm_point_start_listening(struct comm_point* c, int newfd, int msec)
4720 {
4721         verbose(VERB_ALGO, "comm point start listening %d (%d msec)",
4722                 c->fd==-1?newfd:c->fd, msec);
4723         if(c->type == comm_tcp_accept && !c->tcp_free) {
4724                 /* no use to start listening no free slots. */
4725                 return;
4726         }
4727         if(c->event_added) {
4728                 if(ub_event_del(c->ev->ev) != 0) {
4729                         log_err("event_del error to startlisten");
4730                 }
4731                 c->event_added = 0;
4732         }
4733         if(msec != -1 && msec != 0) {
4734                 if(!c->timeout) {
4735                         c->timeout = (struct timeval*)malloc(sizeof(
4736                                 struct timeval));
4737                         if(!c->timeout) {
4738                                 log_err("cpsl: malloc failed. No net read.");
4739                                 return;
4740                         }
4741                 }
4742                 ub_event_add_bits(c->ev->ev, UB_EV_TIMEOUT);
4743 #ifndef S_SPLINT_S /* splint fails on struct timeval. */
4744                 c->timeout->tv_sec = msec/1000;
4745                 c->timeout->tv_usec = (msec%1000)*1000;
4746 #endif /* S_SPLINT_S */
4747         } else {
4748                 if(msec == 0 || !c->timeout) {
4749                         ub_event_del_bits(c->ev->ev, UB_EV_TIMEOUT);
4750                 }
4751         }
4752         if(c->type == comm_tcp || c->type == comm_http) {
4753                 ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
4754                 if(c->tcp_write_and_read) {
4755                         verbose(5, "startlistening %d mode rw", (newfd==-1?c->fd:newfd));
4756                         ub_event_add_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
4757                 } else if(c->tcp_is_reading) {
4758                         verbose(5, "startlistening %d mode r", (newfd==-1?c->fd:newfd));
4759                         ub_event_add_bits(c->ev->ev, UB_EV_READ);
4760                 } else  {
4761                         verbose(5, "startlistening %d mode w", (newfd==-1?c->fd:newfd));
4762                         ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
4763                 }
4764         }
4765         if(newfd != -1) {
4766                 if(c->fd != -1 && c->fd != newfd) {
4767                         verbose(5, "cpsl close of fd %d for %d", c->fd, newfd);
4768                         sock_close(c->fd);
4769                 }
4770                 c->fd = newfd;
4771                 ub_event_set_fd(c->ev->ev, c->fd);
4772         }
4773         if(ub_event_add(c->ev->ev, msec==0?NULL:c->timeout) != 0) {
4774                 log_err("event_add failed. in cpsl.");
4775                 return;
4776         }
4777         c->event_added = 1;
4778 }
4779
4780 void comm_point_listen_for_rw(struct comm_point* c, int rd, int wr)
4781 {
4782         verbose(VERB_ALGO, "comm point listen_for_rw %d %d", c->fd, wr);
4783         if(c->event_added) {
4784                 if(ub_event_del(c->ev->ev) != 0) {
4785                         log_err("event_del error to cplf");
4786                 }
4787                 c->event_added = 0;
4788         }
4789         if(!c->timeout) {
4790                 ub_event_del_bits(c->ev->ev, UB_EV_TIMEOUT);
4791         }
4792         ub_event_del_bits(c->ev->ev, UB_EV_READ|UB_EV_WRITE);
4793         if(rd) ub_event_add_bits(c->ev->ev, UB_EV_READ);
4794         if(wr) ub_event_add_bits(c->ev->ev, UB_EV_WRITE);
4795         if(ub_event_add(c->ev->ev, c->timeout) != 0) {
4796                 log_err("event_add failed. in cplf.");
4797                 return;
4798         }
4799         c->event_added = 1;
4800 }
4801
4802 size_t comm_point_get_mem(struct comm_point* c)
4803 {
4804         size_t s;
4805         if(!c)
4806                 return 0;
4807         s = sizeof(*c) + sizeof(*c->ev);
4808         if(c->timeout)
4809                 s += sizeof(*c->timeout);
4810         if(c->type == comm_tcp || c->type == comm_local) {
4811                 s += sizeof(*c->buffer) + sldns_buffer_capacity(c->buffer);
4812 #ifdef USE_DNSCRYPT
4813                 s += sizeof(*c->dnscrypt_buffer);
4814                 if(c->buffer != c->dnscrypt_buffer) {
4815                         s += sldns_buffer_capacity(c->dnscrypt_buffer);
4816                 }
4817 #endif
4818         }
4819         if(c->type == comm_tcp_accept) {
4820                 int i;
4821                 for(i=0; i<c->max_tcp_count; i++)
4822                         s += comm_point_get_mem(c->tcp_handlers[i]);
4823         }
4824         return s;
4825 }
4826
4827 struct comm_timer*
4828 comm_timer_create(struct comm_base* base, void (*cb)(void*), void* cb_arg)
4829 {
4830         struct internal_timer *tm = (struct internal_timer*)calloc(1,
4831                 sizeof(struct internal_timer));
4832         if(!tm) {
4833                 log_err("malloc failed");
4834                 return NULL;
4835         }
4836         tm->super.ev_timer = tm;
4837         tm->base = base;
4838         tm->super.callback = cb;
4839         tm->super.cb_arg = cb_arg;
4840         tm->ev = ub_event_new(base->eb->base, -1, UB_EV_TIMEOUT,
4841                 comm_timer_callback, &tm->super);
4842         if(tm->ev == NULL) {
4843                 log_err("timer_create: event_base_set failed.");
4844                 free(tm);
4845                 return NULL;
4846         }
4847         return &tm->super;
4848 }
4849
4850 void
4851 comm_timer_disable(struct comm_timer* timer)
4852 {
4853         if(!timer)
4854                 return;
4855         ub_timer_del(timer->ev_timer->ev);
4856         timer->ev_timer->enabled = 0;
4857 }
4858
4859 void
4860 comm_timer_set(struct comm_timer* timer, struct timeval* tv)
4861 {
4862         log_assert(tv);
4863         if(timer->ev_timer->enabled)
4864                 comm_timer_disable(timer);
4865         if(ub_timer_add(timer->ev_timer->ev, timer->ev_timer->base->eb->base,
4866                 comm_timer_callback, timer, tv) != 0)
4867                 log_err("comm_timer_set: evtimer_add failed.");
4868         timer->ev_timer->enabled = 1;
4869 }
4870
4871 void
4872 comm_timer_delete(struct comm_timer* timer)
4873 {
4874         if(!timer)
4875                 return;
4876         comm_timer_disable(timer);
4877         /* Free the sub struct timer->ev_timer derived from the super struct timer.
4878          * i.e. assert(timer == timer->ev_timer)
4879          */
4880         ub_event_free(timer->ev_timer->ev);
4881         free(timer->ev_timer);
4882 }
4883
4884 void
4885 comm_timer_callback(int ATTR_UNUSED(fd), short event, void* arg)
4886 {
4887         struct comm_timer* tm = (struct comm_timer*)arg;
4888         if(!(event&UB_EV_TIMEOUT))
4889                 return;
4890         ub_comm_base_now(tm->ev_timer->base);
4891         tm->ev_timer->enabled = 0;
4892         fptr_ok(fptr_whitelist_comm_timer(tm->callback));
4893         (*tm->callback)(tm->cb_arg);
4894 }
4895
4896 int
4897 comm_timer_is_set(struct comm_timer* timer)
4898 {
4899         return (int)timer->ev_timer->enabled;
4900 }
4901
4902 size_t
4903 comm_timer_get_mem(struct comm_timer* ATTR_UNUSED(timer))
4904 {
4905         return sizeof(struct internal_timer);
4906 }
4907
4908 struct comm_signal*
4909 comm_signal_create(struct comm_base* base,
4910         void (*callback)(int, void*), void* cb_arg)
4911 {
4912         struct comm_signal* com = (struct comm_signal*)malloc(
4913                 sizeof(struct comm_signal));
4914         if(!com) {
4915                 log_err("malloc failed");
4916                 return NULL;
4917         }
4918         com->base = base;
4919         com->callback = callback;
4920         com->cb_arg = cb_arg;
4921         com->ev_signal = NULL;
4922         return com;
4923 }
4924
4925 void
4926 comm_signal_callback(int sig, short event, void* arg)
4927 {
4928         struct comm_signal* comsig = (struct comm_signal*)arg;
4929         if(!(event & UB_EV_SIGNAL))
4930                 return;
4931         ub_comm_base_now(comsig->base);
4932         fptr_ok(fptr_whitelist_comm_signal(comsig->callback));
4933         (*comsig->callback)(sig, comsig->cb_arg);
4934 }
4935
4936 int
4937 comm_signal_bind(struct comm_signal* comsig, int sig)
4938 {
4939         struct internal_signal* entry = (struct internal_signal*)calloc(1,
4940                 sizeof(struct internal_signal));
4941         if(!entry) {
4942                 log_err("malloc failed");
4943                 return 0;
4944         }
4945         log_assert(comsig);
4946         /* add signal event */
4947         entry->ev = ub_signal_new(comsig->base->eb->base, sig,
4948                 comm_signal_callback, comsig);
4949         if(entry->ev == NULL) {
4950                 log_err("Could not create signal event");
4951                 free(entry);
4952                 return 0;
4953         }
4954         if(ub_signal_add(entry->ev, NULL) != 0) {
4955                 log_err("Could not add signal handler");
4956                 ub_event_free(entry->ev);
4957                 free(entry);
4958                 return 0;
4959         }
4960         /* link into list */
4961         entry->next = comsig->ev_signal;
4962         comsig->ev_signal = entry;
4963         return 1;
4964 }
4965
4966 void
4967 comm_signal_delete(struct comm_signal* comsig)
4968 {
4969         struct internal_signal* p, *np;
4970         if(!comsig)
4971                 return;
4972         p=comsig->ev_signal;
4973         while(p) {
4974                 np = p->next;
4975                 ub_signal_del(p->ev);
4976                 ub_event_free(p->ev);
4977                 free(p);
4978                 p = np;
4979         }
4980         free(comsig);
4981 }