contrib/unbound/services/outside_network.c

   1 /*
   2  * services/outside_network.c - implement sending of queries and wait answer.
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file has functions to send queries to authoritative servers and
  40  * wait for the pending answer events.
  41  */
  42 #include "config.h"
  43 #include <ctype.h>
  44 #ifdef HAVE_SYS_TYPES_H
  45 #  include <sys/types.h>
  46 #endif
  47 #include <sys/time.h>
  48 #include "services/outside_network.h"
  49 #include "services/listen_dnsport.h"
  50 #include "services/cache/infra.h"
  51 #include "iterator/iterator.h"
  52 #include "util/data/msgparse.h"
  53 #include "util/data/msgreply.h"
  54 #include "util/data/msgencode.h"
  55 #include "util/data/dname.h"
  56 #include "util/netevent.h"
  57 #include "util/log.h"
  58 #include "util/net_help.h"
  59 #include "util/random.h"
  60 #include "util/fptr_wlist.h"
  61 #include "util/edns.h"
  62 #include "sldns/sbuffer.h"
  63 #include "dnstap/dnstap.h"
  64 #ifdef HAVE_OPENSSL_SSL_H
  65 #include <openssl/ssl.h>
  66 #endif
  67 #ifdef HAVE_X509_VERIFY_PARAM_SET1_HOST
  68 #include <openssl/x509v3.h>
  69 #endif
  70
  71 #ifdef HAVE_NETDB_H
  72 #include <netdb.h>
  73 #endif
  74 #include <fcntl.h>
  75
  76 /** number of times to retry making a random ID that is unique. */
  77 #define MAX_ID_RETRY 1000
  78 /** number of times to retry finding interface, port that can be opened. */
  79 #define MAX_PORT_RETRY 10000
  80 /** number of retries on outgoing UDP queries */
  81 #define OUTBOUND_UDP_RETRY 1
  82
  83 /** initiate TCP transaction for serviced query */
  84 static void serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff);
  85 /** with a fd available, randomize and send UDP */
  86 static int randomize_and_send_udp(struct pending* pend, sldns_buffer* packet,
  87         int timeout);
  88
  89 /** remove waiting tcp from the outnet waiting list */
  90 static void waiting_list_remove(struct outside_network* outnet,
  91         struct waiting_tcp* w);
  92
  93 int
  94 pending_cmp(const void* key1, const void* key2)
  95 {
  96         struct pending *p1 = (struct pending*)key1;
  97         struct pending *p2 = (struct pending*)key2;
  98         if(p1->id < p2->id)
  99                 return -1;
 100         if(p1->id > p2->id)
 101                 return 1;
 102         log_assert(p1->id == p2->id);
 103         return sockaddr_cmp(&p1->addr, p1->addrlen, &p2->addr, p2->addrlen);
 104 }
 105
 106 int
 107 serviced_cmp(const void* key1, const void* key2)
 108 {
 109         struct serviced_query* q1 = (struct serviced_query*)key1;
 110         struct serviced_query* q2 = (struct serviced_query*)key2;
 111         int r;
 112         if(q1->qbuflen < q2->qbuflen)
 113                 return -1;
 114         if(q1->qbuflen > q2->qbuflen)
 115                 return 1;
 116         log_assert(q1->qbuflen == q2->qbuflen);
 117         log_assert(q1->qbuflen >= 15 /* 10 header, root, type, class */);
 118         /* alternate casing of qname is still the same query */
 119         if((r = memcmp(q1->qbuf, q2->qbuf, 10)) != 0)
 120                 return r;
 121         if((r = memcmp(q1->qbuf+q1->qbuflen-4, q2->qbuf+q2->qbuflen-4, 4)) != 0)
 122                 return r;
 123         if(q1->dnssec != q2->dnssec) {
 124                 if(q1->dnssec < q2->dnssec)
 125                         return -1;
 126                 return 1;
 127         }
 128         if((r = query_dname_compare(q1->qbuf+10, q2->qbuf+10)) != 0)
 129                 return r;
 130         if((r = edns_opt_list_compare(q1->opt_list, q2->opt_list)) != 0)
 131                 return r;
 132         return sockaddr_cmp(&q1->addr, q1->addrlen, &q2->addr, q2->addrlen);
 133 }
 134
 135 /** compare if the reuse element has the same address, port and same ssl-is
 136  * used-for-it characteristic */
 137 static int
 138 reuse_cmp_addrportssl(const void* key1, const void* key2)
 139 {
 140         struct reuse_tcp* r1 = (struct reuse_tcp*)key1;
 141         struct reuse_tcp* r2 = (struct reuse_tcp*)key2;
 142         int r;
 143         /* compare address and port */
 144         r = sockaddr_cmp(&r1->addr, r1->addrlen, &r2->addr, r2->addrlen);
 145         if(r != 0)
 146                 return r;
 147
 148         /* compare if SSL-enabled */
 149         if(r1->is_ssl && !r2->is_ssl)
 150                 return 1;
 151         if(!r1->is_ssl && r2->is_ssl)
 152                 return -1;
 153         return 0;
 154 }
 155
 156 int
 157 reuse_cmp(const void* key1, const void* key2)
 158 {
 159         int r;
 160         r = reuse_cmp_addrportssl(key1, key2);
 161         if(r != 0)
 162                 return r;
 163
 164         /* compare ptr value */
 165         if(key1 < key2) return -1;
 166         if(key1 > key2) return 1;
 167         return 0;
 168 }
 169
 170 int reuse_id_cmp(const void* key1, const void* key2)
 171 {
 172         struct waiting_tcp* w1 = (struct waiting_tcp*)key1;
 173         struct waiting_tcp* w2 = (struct waiting_tcp*)key2;
 174         if(w1->id < w2->id)
 175                 return -1;
 176         if(w1->id > w2->id)
 177                 return 1;
 178         return 0;
 179 }
 180
 181 /** delete waiting_tcp entry. Does not unlink from waiting list.
 182  * @param w: to delete.
 183  */
 184 static void
 185 waiting_tcp_delete(struct waiting_tcp* w)
 186 {
 187         if(!w) return;
 188         if(w->timer)
 189                 comm_timer_delete(w->timer);
 190         free(w);
 191 }
 192
 193 /**
 194  * Pick random outgoing-interface of that family, and bind it.
 195  * port set to 0 so OS picks a port number for us.
 196  * if it is the ANY address, do not bind.
 197  * @param w: tcp structure with destination address.
 198  * @param s: socket fd.
 199  * @return false on error, socket closed.
 200  */
 201 static int
 202 pick_outgoing_tcp(struct waiting_tcp* w, int s)
 203 {
 204         struct port_if* pi = NULL;
 205         int num;
 206 #ifdef INET6
 207         if(addr_is_ip6(&w->addr, w->addrlen))
 208                 num = w->outnet->num_ip6;
 209         else
 210 #endif
 211                 num = w->outnet->num_ip4;
 212         if(num == 0) {
 213                 log_err("no TCP outgoing interfaces of family");
 214                 log_addr(VERB_OPS, "for addr", &w->addr, w->addrlen);
 215                 sock_close(s);
 216                 return 0;
 217         }
 218 #ifdef INET6
 219         if(addr_is_ip6(&w->addr, w->addrlen))
 220                 pi = &w->outnet->ip6_ifs[ub_random_max(w->outnet->rnd, num)];
 221         else
 222 #endif
 223                 pi = &w->outnet->ip4_ifs[ub_random_max(w->outnet->rnd, num)];
 224         log_assert(pi);
 225         if(addr_is_any(&pi->addr, pi->addrlen)) {
 226                 /* binding to the ANY interface is for listening sockets */
 227                 return 1;
 228         }
 229         /* set port to 0 */
 230         if(addr_is_ip6(&pi->addr, pi->addrlen))
 231                 ((struct sockaddr_in6*)&pi->addr)->sin6_port = 0;
 232         else    ((struct sockaddr_in*)&pi->addr)->sin_port = 0;
 233         if(bind(s, (struct sockaddr*)&pi->addr, pi->addrlen) != 0) {
 234                 log_err("outgoing tcp: bind: %s", sock_strerror(errno));
 235                 sock_close(s);
 236                 return 0;
 237         }
 238         log_addr(VERB_ALGO, "tcp bound to src", &pi->addr, pi->addrlen);
 239         return 1;
 240 }
 241
 242 /** get TCP file descriptor for address, returns -1 on failure,
 243  * tcp_mss is 0 or maxseg size to set for TCP packets. */
 244 int
 245 outnet_get_tcp_fd(struct sockaddr_storage* addr, socklen_t addrlen, int tcp_mss, int dscp)
 246 {
 247         int s;
 248         int af;
 249         char* err;
 250 #ifdef SO_REUSEADDR
 251         int on = 1;
 252 #endif
 253 #ifdef INET6
 254         if(addr_is_ip6(addr, addrlen)){
 255                 s = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
 256                 af = AF_INET6;
 257         } else {
 258 #else
 259         {
 260 #endif
 261                 af = AF_INET;
 262                 s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 263         }
 264         if(s == -1) {
 265                 log_err_addr("outgoing tcp: socket", sock_strerror(errno),
 266                         addr, addrlen);
 267                 return -1;
 268         }
 269
 270 #ifdef SO_REUSEADDR
 271         if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on,
 272                 (socklen_t)sizeof(on)) < 0) {
 273                 verbose(VERB_ALGO, "outgoing tcp:"
 274                         " setsockopt(.. SO_REUSEADDR ..) failed");
 275         }
 276 #endif
 277
 278         err = set_ip_dscp(s, af, dscp);
 279         if(err != NULL) {
 280                 verbose(VERB_ALGO, "outgoing tcp:"
 281                         "error setting IP DiffServ codepoint on socket");
 282         }
 283
 284         if(tcp_mss > 0) {
 285 #if defined(IPPROTO_TCP) && defined(TCP_MAXSEG)
 286                 if(setsockopt(s, IPPROTO_TCP, TCP_MAXSEG,
 287                         (void*)&tcp_mss, (socklen_t)sizeof(tcp_mss)) < 0) {
 288                         verbose(VERB_ALGO, "outgoing tcp:"
 289                                 " setsockopt(.. TCP_MAXSEG ..) failed");
 290                 }
 291 #else
 292                 verbose(VERB_ALGO, "outgoing tcp:"
 293                         " setsockopt(TCP_MAXSEG) unsupported");
 294 #endif /* defined(IPPROTO_TCP) && defined(TCP_MAXSEG) */
 295         }
 296
 297         return s;
 298 }
 299
 300 /** connect tcp connection to addr, 0 on failure */
 301 int
 302 outnet_tcp_connect(int s, struct sockaddr_storage* addr, socklen_t addrlen)
 303 {
 304         if(connect(s, (struct sockaddr*)addr, addrlen) == -1) {
 305 #ifndef USE_WINSOCK
 306 #ifdef EINPROGRESS
 307                 if(errno != EINPROGRESS) {
 308 #endif
 309                         if(tcp_connect_errno_needs_log(
 310                                 (struct sockaddr*)addr, addrlen))
 311                                 log_err_addr("outgoing tcp: connect",
 312                                         strerror(errno), addr, addrlen);
 313                         close(s);
 314                         return 0;
 315 #ifdef EINPROGRESS
 316                 }
 317 #endif
 318 #else /* USE_WINSOCK */
 319                 if(WSAGetLastError() != WSAEINPROGRESS &&
 320                         WSAGetLastError() != WSAEWOULDBLOCK) {
 321                         closesocket(s);
 322                         return 0;
 323                 }
 324 #endif
 325         }
 326         return 1;
 327 }
 328
 329 /** log reuse item addr and ptr with message */
 330 static void
 331 log_reuse_tcp(enum verbosity_value v, const char* msg, struct reuse_tcp* reuse)
 332 {
 333         uint16_t port;
 334         char addrbuf[128];
 335         if(verbosity < v) return;
 336         addr_to_str(&reuse->addr, reuse->addrlen, addrbuf, sizeof(addrbuf));
 337         port = ntohs(((struct sockaddr_in*)&reuse->addr)->sin_port);
 338         verbose(v, "%s %s#%u fd %d", msg, addrbuf, (unsigned)port,
 339                 reuse->pending->c->fd);
 340 }
 341
 342 /** pop the first element from the writewait list */
 343 static struct waiting_tcp* reuse_write_wait_pop(struct reuse_tcp* reuse)
 344 {
 345         struct waiting_tcp* w = reuse->write_wait_first;
 346         if(!w)
 347                 return NULL;
 348         log_assert(w->write_wait_queued);
 349         log_assert(!w->write_wait_prev);
 350         reuse->write_wait_first = w->write_wait_next;
 351         if(w->write_wait_next)
 352                 w->write_wait_next->write_wait_prev = NULL;
 353         else    reuse->write_wait_last = NULL;
 354         w->write_wait_queued = 0;
 355         return w;
 356 }
 357
 358 /** remove the element from the writewait list */
 359 static void reuse_write_wait_remove(struct reuse_tcp* reuse,
 360         struct waiting_tcp* w)
 361 {
 362         if(!w)
 363                 return;
 364         if(!w->write_wait_queued)
 365                 return;
 366         if(w->write_wait_prev)
 367                 w->write_wait_prev->write_wait_next = w->write_wait_next;
 368         else    reuse->write_wait_first = w->write_wait_next;
 369         if(w->write_wait_next)
 370                 w->write_wait_next->write_wait_prev = w->write_wait_prev;
 371         else    reuse->write_wait_last = w->write_wait_prev;
 372         w->write_wait_queued = 0;
 373 }
 374
 375 /** push the element after the last on the writewait list */
 376 static void reuse_write_wait_push_back(struct reuse_tcp* reuse,
 377         struct waiting_tcp* w)
 378 {
 379         if(!w) return;
 380         log_assert(!w->write_wait_queued);
 381         if(reuse->write_wait_last) {
 382                 reuse->write_wait_last->write_wait_next = w;
 383                 w->write_wait_prev = reuse->write_wait_last;
 384         } else {
 385                 reuse->write_wait_first = w;
 386         }
 387         reuse->write_wait_last = w;
 388         w->write_wait_queued = 1;
 389 }
 390
 391 /** insert element in tree by id */
 392 void
 393 reuse_tree_by_id_insert(struct reuse_tcp* reuse, struct waiting_tcp* w)
 394 {
 395         log_assert(w->id_node.key == NULL);
 396         w->id_node.key = w;
 397         rbtree_insert(&reuse->tree_by_id, &w->id_node);
 398 }
 399
 400 /** find element in tree by id */
 401 struct waiting_tcp*
 402 reuse_tcp_by_id_find(struct reuse_tcp* reuse, uint16_t id)
 403 {
 404         struct waiting_tcp key_w;
 405         rbnode_type* n;
 406         memset(&key_w, 0, sizeof(key_w));
 407         key_w.id_node.key = &key_w;
 408         key_w.id = id;
 409         n = rbtree_search(&reuse->tree_by_id, &key_w);
 410         if(!n) return NULL;
 411         return (struct waiting_tcp*)n->key;
 412 }
 413
 414 /** return ID value of rbnode in tree_by_id */
 415 static uint16_t
 416 tree_by_id_get_id(rbnode_type* node)
 417 {
 418         struct waiting_tcp* w = (struct waiting_tcp*)node->key;
 419         return w->id;
 420 }
 421
 422 /** insert into reuse tcp tree and LRU, false on failure (duplicate) */
 423 static int
 424 reuse_tcp_insert(struct outside_network* outnet, struct pending_tcp* pend_tcp)
 425 {
 426         log_reuse_tcp(VERB_CLIENT, "reuse_tcp_insert", &pend_tcp->reuse);
 427         if(pend_tcp->reuse.item_on_lru_list)
 428                 return 1;
 429         pend_tcp->reuse.node.key = &pend_tcp->reuse;
 430         pend_tcp->reuse.pending = pend_tcp;
 431         if(!rbtree_insert(&outnet->tcp_reuse, &pend_tcp->reuse.node)) {
 432                 /* this is a duplicate connection, close this one */
 433                 verbose(VERB_CLIENT, "reuse_tcp_insert: duplicate connection");
 434                 pend_tcp->reuse.node.key = NULL;
 435                 return 0;
 436         }
 437         /* insert into LRU, first is newest */
 438         pend_tcp->reuse.lru_prev = NULL;
 439         if(outnet->tcp_reuse_first) {
 440                 pend_tcp->reuse.lru_next = outnet->tcp_reuse_first;
 441                 outnet->tcp_reuse_first->lru_prev = &pend_tcp->reuse;
 442         } else {
 443                 pend_tcp->reuse.lru_next = NULL;
 444                 outnet->tcp_reuse_last = &pend_tcp->reuse;
 445         }
 446         outnet->tcp_reuse_first = &pend_tcp->reuse;
 447         pend_tcp->reuse.item_on_lru_list = 1;
 448         return 1;
 449 }
 450
 451 /** find reuse tcp stream to destination for query, or NULL if none */
 452 static struct reuse_tcp*
 453 reuse_tcp_find(struct outside_network* outnet, struct sockaddr_storage* addr,
 454         socklen_t addrlen, int use_ssl)
 455 {
 456         struct waiting_tcp key_w;
 457         struct pending_tcp key_p;
 458         struct comm_point c;
 459         rbnode_type* result = NULL, *prev;
 460         verbose(VERB_CLIENT, "reuse_tcp_find");
 461         memset(&key_w, 0, sizeof(key_w));
 462         memset(&key_p, 0, sizeof(key_p));
 463         memset(&c, 0, sizeof(c));
 464         key_p.query = &key_w;
 465         key_p.c = &c;
 466         key_p.reuse.pending = &key_p;
 467         key_p.reuse.node.key = &key_p.reuse;
 468         if(use_ssl)
 469                 key_p.reuse.is_ssl = 1;
 470         if(addrlen > (socklen_t)sizeof(key_p.reuse.addr))
 471                 return NULL;
 472         memmove(&key_p.reuse.addr, addr, addrlen);
 473         key_p.reuse.addrlen = addrlen;
 474
 475         verbose(VERB_CLIENT, "reuse_tcp_find: num reuse streams %u",
 476                 (unsigned)outnet->tcp_reuse.count);
 477         if(outnet->tcp_reuse.root == NULL ||
 478                 outnet->tcp_reuse.root == RBTREE_NULL)
 479                 return NULL;
 480         if(rbtree_find_less_equal(&outnet->tcp_reuse, &key_p.reuse.node,
 481                 &result)) {
 482                 /* exact match */
 483                 /* but the key is on stack, and ptr is compared, impossible */
 484                 log_assert(&key_p.reuse != (struct reuse_tcp*)result);
 485                 log_assert(&key_p != ((struct reuse_tcp*)result)->pending);
 486         }
 487         /* not found, return null */
 488         if(!result || result == RBTREE_NULL)
 489                 return NULL;
 490         verbose(VERB_CLIENT, "reuse_tcp_find check inexact match");
 491         /* inexact match, find one of possibly several connections to the
 492          * same destination address, with the correct port, ssl, and
 493          * also less than max number of open queries, or else, fail to open
 494          * a new one */
 495         /* rewind to start of sequence of same address,port,ssl */
 496         prev = rbtree_previous(result);
 497         while(prev && prev != RBTREE_NULL &&
 498                 reuse_cmp_addrportssl(prev->key, &key_p.reuse) == 0) {
 499                 result = prev;
 500                 prev = rbtree_previous(result);
 501         }
 502
 503         /* loop to find first one that has correct characteristics */
 504         while(result && result != RBTREE_NULL &&
 505                 reuse_cmp_addrportssl(result->key, &key_p.reuse) == 0) {
 506                 if(((struct reuse_tcp*)result)->tree_by_id.count <
 507                         MAX_REUSE_TCP_QUERIES) {
 508                         /* same address, port, ssl-yes-or-no, and has
 509                          * space for another query */
 510                         return (struct reuse_tcp*)result;
 511                 }
 512                 result = rbtree_next(result);
 513         }
 514         return NULL;
 515 }
 516
 517 /** use the buffer to setup writing the query */
 518 static void
 519 outnet_tcp_take_query_setup(int s, struct pending_tcp* pend,
 520         struct waiting_tcp* w)
 521 {
 522         struct timeval tv;
 523         verbose(VERB_CLIENT, "outnet_tcp_take_query_setup: setup packet to write "
 524                 "len %d timeout %d msec",
 525                 (int)w->pkt_len, w->timeout);
 526         pend->c->tcp_write_pkt = w->pkt;
 527         pend->c->tcp_write_pkt_len = w->pkt_len;
 528         pend->c->tcp_write_and_read = 1;
 529         pend->c->tcp_write_byte_count = 0;
 530         pend->c->tcp_is_reading = 0;
 531         comm_point_start_listening(pend->c, s, -1);
 532         /* set timer on the waiting_tcp entry, this is the write timeout
 533          * for the written packet.  The timer on pend->c is the timer
 534          * for when there is no written packet and we have readtimeouts */
 535 #ifndef S_SPLINT_S
 536         tv.tv_sec = w->timeout/1000;
 537         tv.tv_usec = (w->timeout%1000)*1000;
 538 #endif
 539         /* if the waiting_tcp was previously waiting for a buffer in the
 540          * outside_network.tcpwaitlist, then the timer is reset now that
 541          * we start writing it */
 542         comm_timer_set(w->timer, &tv);
 543 }
 544
 545 /** use next free buffer to service a tcp query */
 546 static int
 547 outnet_tcp_take_into_use(struct waiting_tcp* w)
 548 {
 549         struct pending_tcp* pend = w->outnet->tcp_free;
 550         int s;
 551         log_assert(pend);
 552         log_assert(w->pkt);
 553         log_assert(w->pkt_len > 0);
 554         log_assert(w->addrlen > 0);
 555         pend->c->tcp_do_toggle_rw = 0;
 556         pend->c->tcp_do_close = 0;
 557         /* open socket */
 558         s = outnet_get_tcp_fd(&w->addr, w->addrlen, w->outnet->tcp_mss, w->outnet->ip_dscp);
 559
 560         if(s == -1)
 561                 return 0;
 562
 563         if(!pick_outgoing_tcp(w, s))
 564                 return 0;
 565
 566         fd_set_nonblock(s);
 567 #ifdef USE_OSX_MSG_FASTOPEN
 568         /* API for fast open is different here. We use a connectx() function and
 569            then writes can happen as normal even using SSL.*/
 570         /* connectx requires that the len be set in the sockaddr struct*/
 571         struct sockaddr_in *addr_in = (struct sockaddr_in *)&w->addr;
 572         addr_in->sin_len = w->addrlen;
 573         sa_endpoints_t endpoints;
 574         endpoints.sae_srcif = 0;
 575         endpoints.sae_srcaddr = NULL;
 576         endpoints.sae_srcaddrlen = 0;
 577         endpoints.sae_dstaddr = (struct sockaddr *)&w->addr;
 578         endpoints.sae_dstaddrlen = w->addrlen;
 579         if (connectx(s, &endpoints, SAE_ASSOCID_ANY,
 580                      CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE,
 581                      NULL, 0, NULL, NULL) == -1) {
 582                 /* if fails, failover to connect for OSX 10.10 */
 583 #ifdef EINPROGRESS
 584                 if(errno != EINPROGRESS) {
 585 #else
 586                 if(1) {
 587 #endif
 588                         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 589 #else /* USE_OSX_MSG_FASTOPEN*/
 590 #ifdef USE_MSG_FASTOPEN
 591         pend->c->tcp_do_fastopen = 1;
 592         /* Only do TFO for TCP in which case no connect() is required here.
 593            Don't combine client TFO with SSL, since OpenSSL can't
 594            currently support doing a handshake on fd that already isn't connected*/
 595         if (w->outnet->sslctx && w->ssl_upstream) {
 596                 if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 597 #else /* USE_MSG_FASTOPEN*/
 598         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 599 #endif /* USE_MSG_FASTOPEN*/
 600 #endif /* USE_OSX_MSG_FASTOPEN*/
 601 #ifndef USE_WINSOCK
 602 #ifdef EINPROGRESS
 603                 if(errno != EINPROGRESS) {
 604 #else
 605                 if(1) {
 606 #endif
 607                         if(tcp_connect_errno_needs_log(
 608                                 (struct sockaddr*)&w->addr, w->addrlen))
 609                                 log_err_addr("outgoing tcp: connect",
 610                                         strerror(errno), &w->addr, w->addrlen);
 611                         close(s);
 612 #else /* USE_WINSOCK */
 613                 if(WSAGetLastError() != WSAEINPROGRESS &&
 614                         WSAGetLastError() != WSAEWOULDBLOCK) {
 615                         closesocket(s);
 616 #endif
 617                         return 0;
 618                 }
 619         }
 620 #ifdef USE_MSG_FASTOPEN
 621         }
 622 #endif /* USE_MSG_FASTOPEN */
 623 #ifdef USE_OSX_MSG_FASTOPEN
 624                 }
 625         }
 626 #endif /* USE_OSX_MSG_FASTOPEN */
 627         if(w->outnet->sslctx && w->ssl_upstream) {
 628                 pend->c->ssl = outgoing_ssl_fd(w->outnet->sslctx, s);
 629                 if(!pend->c->ssl) {
 630                         pend->c->fd = s;
 631                         comm_point_close(pend->c);
 632                         return 0;
 633                 }
 634                 verbose(VERB_ALGO, "the query is using TLS encryption, for %s",
 635                         (w->tls_auth_name?w->tls_auth_name:"an unauthenticated connection"));
 636 #ifdef USE_WINSOCK
 637                 comm_point_tcp_win_bio_cb(pend->c, pend->c->ssl);
 638 #endif
 639                 pend->c->ssl_shake_state = comm_ssl_shake_write;
 640                 if(!set_auth_name_on_ssl(pend->c->ssl, w->tls_auth_name,
 641                         w->outnet->tls_use_sni)) {
 642                         pend->c->fd = s;
 643 #ifdef HAVE_SSL
 644                         SSL_free(pend->c->ssl);
 645 #endif
 646                         pend->c->ssl = NULL;
 647                         comm_point_close(pend->c);
 648                         return 0;
 649                 }
 650         }
 651         w->next_waiting = (void*)pend;
 652         w->outnet->num_tcp_outgoing++;
 653         w->outnet->tcp_free = pend->next_free;
 654         pend->next_free = NULL;
 655         pend->query = w;
 656         pend->reuse.outnet = w->outnet;
 657         pend->c->repinfo.addrlen = w->addrlen;
 658         pend->c->tcp_more_read_again = &pend->reuse.cp_more_read_again;
 659         pend->c->tcp_more_write_again = &pend->reuse.cp_more_write_again;
 660         pend->reuse.cp_more_read_again = 0;
 661         pend->reuse.cp_more_write_again = 0;
 662         memcpy(&pend->c->repinfo.addr, &w->addr, w->addrlen);
 663         pend->reuse.pending = pend;
 664         if(pend->c->ssl)
 665                 pend->reuse.is_ssl = 1;
 666         else    pend->reuse.is_ssl = 0;
 667         /* insert in reuse by address tree if not already inserted there */
 668         (void)reuse_tcp_insert(w->outnet, pend);
 669         reuse_tree_by_id_insert(&pend->reuse, w);
 670         outnet_tcp_take_query_setup(s, pend, w);
 671         return 1;
 672 }
 673
 674 /** Touch the lru of a reuse_tcp element, it is in use.
 675  * This moves it to the front of the list, where it is not likely to
 676  * be closed.  Items at the back of the list are closed to make space. */
 677 static void
 678 reuse_tcp_lru_touch(struct outside_network* outnet, struct reuse_tcp* reuse)
 679 {
 680         if(!reuse->item_on_lru_list)
 681                 return; /* not on the list, no lru to modify */
 682         if(!reuse->lru_prev)
 683                 return; /* already first in the list */
 684         /* remove at current position */
 685         /* since it is not first, there is a previous element */
 686         reuse->lru_prev->lru_next = reuse->lru_next;
 687         if(reuse->lru_next)
 688                 reuse->lru_next->lru_prev = reuse->lru_prev;
 689         else    outnet->tcp_reuse_last = reuse->lru_prev;
 690         /* insert at the front */
 691         reuse->lru_prev = NULL;
 692         reuse->lru_next = outnet->tcp_reuse_first;
 693         /* since it is not first, it is not the only element and
 694          * lru_next is thus not NULL and thus reuse is now not the last in
 695          * the list, so outnet->tcp_reuse_last does not need to be modified */
 696         outnet->tcp_reuse_first = reuse;
 697 }
 698
 699 /** call callback on waiting_tcp, if not NULL */
 700 static void
 701 waiting_tcp_callback(struct waiting_tcp* w, struct comm_point* c, int error,
 702         struct comm_reply* reply_info)
 703 {
 704         if(w->cb) {
 705                 fptr_ok(fptr_whitelist_pending_tcp(w->cb));
 706                 (void)(*w->cb)(c, w->cb_arg, error, reply_info);
 707         }
 708 }
 709
 710 /** see if buffers can be used to service TCP queries */
 711 static void
 712 use_free_buffer(struct outside_network* outnet)
 713 {
 714         struct waiting_tcp* w;
 715         while(outnet->tcp_free && outnet->tcp_wait_first
 716                 && !outnet->want_to_quit) {
 717                 struct reuse_tcp* reuse = NULL;
 718                 w = outnet->tcp_wait_first;
 719                 outnet->tcp_wait_first = w->next_waiting;
 720                 if(outnet->tcp_wait_last == w)
 721                         outnet->tcp_wait_last = NULL;
 722                 w->on_tcp_waiting_list = 0;
 723                 reuse = reuse_tcp_find(outnet, &w->addr, w->addrlen,
 724                         w->ssl_upstream);
 725                 if(reuse) {
 726                         log_reuse_tcp(VERB_CLIENT, "use free buffer for waiting tcp: "
 727                                 "found reuse", reuse);
 728                         reuse_tcp_lru_touch(outnet, reuse);
 729                         comm_timer_disable(w->timer);
 730                         w->next_waiting = (void*)reuse->pending;
 731                         reuse_tree_by_id_insert(reuse, w);
 732                         if(reuse->pending->query) {
 733                                 /* on the write wait list */
 734                                 reuse_write_wait_push_back(reuse, w);
 735                         } else {
 736                                 /* write straight away */
 737                                 /* stop the timer on read of the fd */
 738                                 comm_point_stop_listening(reuse->pending->c);
 739                                 reuse->pending->query = w;
 740                                 outnet_tcp_take_query_setup(
 741                                         reuse->pending->c->fd, reuse->pending,
 742                                         w);
 743                         }
 744                 } else {
 745                         struct pending_tcp* pend = w->outnet->tcp_free;
 746                         rbtree_init(&pend->reuse.tree_by_id, reuse_id_cmp);
 747                         pend->reuse.pending = pend;
 748                         memcpy(&pend->reuse.addr, &w->addr, w->addrlen);
 749                         pend->reuse.addrlen = w->addrlen;
 750                         if(!outnet_tcp_take_into_use(w)) {
 751                                 waiting_tcp_callback(w, NULL, NETEVENT_CLOSED,
 752                                         NULL);
 753                                 waiting_tcp_delete(w);
 754                         }
 755                 }
 756         }
 757 }
 758
 759 /** add waiting_tcp element to the outnet tcp waiting list */
 760 static void
 761 outnet_add_tcp_waiting(struct outside_network* outnet, struct waiting_tcp* w)
 762 {
 763         struct timeval tv;
 764         if(w->on_tcp_waiting_list)
 765                 return;
 766         w->next_waiting = NULL;
 767         if(outnet->tcp_wait_last)
 768                 outnet->tcp_wait_last->next_waiting = w;
 769         else    outnet->tcp_wait_first = w;
 770         outnet->tcp_wait_last = w;
 771         w->on_tcp_waiting_list = 1;
 772 #ifndef S_SPLINT_S
 773         tv.tv_sec = w->timeout/1000;
 774         tv.tv_usec = (w->timeout%1000)*1000;
 775 #endif
 776         comm_timer_set(w->timer, &tv);
 777 }
 778
 779 /** delete element from tree by id */
 780 static void
 781 reuse_tree_by_id_delete(struct reuse_tcp* reuse, struct waiting_tcp* w)
 782 {
 783         log_assert(w->id_node.key != NULL);
 784         rbtree_delete(&reuse->tree_by_id, w);
 785         w->id_node.key = NULL;
 786 }
 787
 788 /** move writewait list to go for another connection. */
 789 static void
 790 reuse_move_writewait_away(struct outside_network* outnet,
 791         struct pending_tcp* pend)
 792 {
 793         /* the writewait list has not been written yet, so if the
 794          * stream was closed, they have not actually been failed, only
 795          * the queries written.  Other queries can get written to another
 796          * stream.  For upstreams that do not support multiple queries
 797          * and answers, the stream can get closed, and then the queries
 798          * can get written on a new socket */
 799         struct waiting_tcp* w;
 800         if(pend->query && pend->query->error_count == 0 &&
 801                 pend->c->tcp_write_pkt == pend->query->pkt &&
 802                 pend->c->tcp_write_pkt_len == pend->query->pkt_len) {
 803                 /* since the current query is not written, it can also
 804                  * move to a free buffer */
 805                 if(verbosity >= VERB_CLIENT && pend->query->pkt_len > 12+2+2 &&
 806                         LDNS_QDCOUNT(pend->query->pkt) > 0 &&
 807                         dname_valid(pend->query->pkt+12, pend->query->pkt_len-12)) {
 808                         char buf[LDNS_MAX_DOMAINLEN+1];
 809                         dname_str(pend->query->pkt+12, buf);
 810                         verbose(VERB_CLIENT, "reuse_move_writewait_away current %s %d bytes were written",
 811                                 buf, (int)pend->c->tcp_write_byte_count);
 812                 }
 813                 pend->c->tcp_write_pkt = NULL;
 814                 pend->c->tcp_write_pkt_len = 0;
 815                 pend->c->tcp_write_and_read = 0;
 816                 pend->reuse.cp_more_read_again = 0;
 817                 pend->reuse.cp_more_write_again = 0;
 818                 pend->c->tcp_is_reading = 1;
 819                 w = pend->query;
 820                 pend->query = NULL;
 821                 /* increase error count, so that if the next socket fails too
 822                  * the server selection is run again with this query failed
 823                  * and it can select a different server (if possible), or
 824                  * fail the query */
 825                 w->error_count ++;
 826                 reuse_tree_by_id_delete(&pend->reuse, w);
 827                 outnet_add_tcp_waiting(outnet, w);
 828         }
 829         while((w = reuse_write_wait_pop(&pend->reuse)) != NULL) {
 830                 if(verbosity >= VERB_CLIENT && w->pkt_len > 12+2+2 &&
 831                         LDNS_QDCOUNT(w->pkt) > 0 &&
 832                         dname_valid(w->pkt+12, w->pkt_len-12)) {
 833                         char buf[LDNS_MAX_DOMAINLEN+1];
 834                         dname_str(w->pkt+12, buf);
 835                         verbose(VERB_CLIENT, "reuse_move_writewait_away item %s", buf);
 836                 }
 837                 reuse_tree_by_id_delete(&pend->reuse, w);
 838                 outnet_add_tcp_waiting(outnet, w);
 839         }
 840 }
 841
 842 /** remove reused element from tree and lru list */
 843 static void
 844 reuse_tcp_remove_tree_list(struct outside_network* outnet,
 845         struct reuse_tcp* reuse)
 846 {
 847         verbose(VERB_CLIENT, "reuse_tcp_remove_tree_list");
 848         if(reuse->node.key) {
 849                 /* delete it from reuse tree */
 850                 (void)rbtree_delete(&outnet->tcp_reuse, &reuse->node);
 851                 reuse->node.key = NULL;
 852         }
 853         /* delete from reuse list */
 854         if(reuse->item_on_lru_list) {
 855                 if(reuse->lru_prev) {
 856                         /* assert that members of the lru list are waiting
 857                          * and thus have a pending pointer to the struct */
 858                         log_assert(reuse->lru_prev->pending);
 859                         reuse->lru_prev->lru_next = reuse->lru_next;
 860                 } else {
 861                         log_assert(!reuse->lru_next || reuse->lru_next->pending);
 862                         outnet->tcp_reuse_first = reuse->lru_next;
 863                 }
 864                 if(reuse->lru_next) {
 865                         /* assert that members of the lru list are waiting
 866                          * and thus have a pending pointer to the struct */
 867                         log_assert(reuse->lru_next->pending);
 868                         reuse->lru_next->lru_prev = reuse->lru_prev;
 869                 } else {
 870                         log_assert(!reuse->lru_prev || reuse->lru_prev->pending);
 871                         outnet->tcp_reuse_last = reuse->lru_prev;
 872                 }
 873                 reuse->item_on_lru_list = 0;
 874         }
 875 }
 876
 877 /** helper function that deletes an element from the tree of readwait
 878  * elements in tcp reuse structure */
 879 static void reuse_del_readwait_elem(rbnode_type* node, void* ATTR_UNUSED(arg))
 880 {
 881         struct waiting_tcp* w = (struct waiting_tcp*)node->key;
 882         waiting_tcp_delete(w);
 883 }
 884
 885 /** delete readwait waiting_tcp elements, deletes the elements in the list */
 886 void reuse_del_readwait(rbtree_type* tree_by_id)
 887 {
 888         if(tree_by_id->root == NULL ||
 889                 tree_by_id->root == RBTREE_NULL)
 890                 return;
 891         traverse_postorder(tree_by_id, &reuse_del_readwait_elem, NULL);
 892         rbtree_init(tree_by_id, reuse_id_cmp);
 893 }
 894
 895 /** decommission a tcp buffer, closes commpoint and frees waiting_tcp entry */
 896 static void
 897 decommission_pending_tcp(struct outside_network* outnet,
 898         struct pending_tcp* pend)
 899 {
 900         verbose(VERB_CLIENT, "decommission_pending_tcp");
 901         pend->next_free = outnet->tcp_free;
 902         outnet->tcp_free = pend;
 903         if(pend->reuse.node.key) {
 904                 /* needs unlink from the reuse tree to get deleted */
 905                 reuse_tcp_remove_tree_list(outnet, &pend->reuse);
 906         }
 907         /* free SSL structure after remove from outnet tcp reuse tree,
 908          * because the c->ssl null or not is used for sorting in the tree */
 909         if(pend->c->ssl) {
 910 #ifdef HAVE_SSL
 911                 SSL_shutdown(pend->c->ssl);
 912                 SSL_free(pend->c->ssl);
 913                 pend->c->ssl = NULL;
 914 #endif
 915         }
 916         comm_point_close(pend->c);
 917         pend->reuse.cp_more_read_again = 0;
 918         pend->reuse.cp_more_write_again = 0;
 919         /* unlink the query and writewait list, it is part of the tree
 920          * nodes and is deleted */
 921         pend->query = NULL;
 922         pend->reuse.write_wait_first = NULL;
 923         pend->reuse.write_wait_last = NULL;
 924         reuse_del_readwait(&pend->reuse.tree_by_id);
 925 }
 926
 927 /** perform failure callbacks for waiting queries in reuse read rbtree */
 928 static void reuse_cb_readwait_for_failure(rbtree_type* tree_by_id, int err)
 929 {
 930         rbnode_type* node;
 931         if(tree_by_id->root == NULL ||
 932                 tree_by_id->root == RBTREE_NULL)
 933                 return;
 934         node = rbtree_first(tree_by_id);
 935         while(node && node != RBTREE_NULL) {
 936                 struct waiting_tcp* w = (struct waiting_tcp*)node->key;
 937                 waiting_tcp_callback(w, NULL, err, NULL);
 938                 node = rbtree_next(node);
 939         }
 940 }
 941
 942 /** perform callbacks for failure and also decommission pending tcp.
 943  * the callbacks remove references in sq->pending to the waiting_tcp
 944  * members of the tree_by_id in the pending tcp.  The pending_tcp is
 945  * removed before the callbacks, so that the callbacks do not modify
 946  * the pending_tcp due to its reference in the outside_network reuse tree */
 947 static void reuse_cb_and_decommission(struct outside_network* outnet,
 948         struct pending_tcp* pend, int error)
 949 {
 950         rbtree_type store;
 951         store = pend->reuse.tree_by_id;
 952         pend->query = NULL;
 953         rbtree_init(&pend->reuse.tree_by_id, reuse_id_cmp);
 954         pend->reuse.write_wait_first = NULL;
 955         pend->reuse.write_wait_last = NULL;
 956         decommission_pending_tcp(outnet, pend);
 957         reuse_cb_readwait_for_failure(&store, error);
 958         reuse_del_readwait(&store);
 959 }
 960
 961 /** set timeout on tcp fd and setup read event to catch incoming dns msgs */
 962 static void
 963 reuse_tcp_setup_timeout(struct pending_tcp* pend_tcp)
 964 {
 965         log_reuse_tcp(VERB_CLIENT, "reuse_tcp_setup_timeout", &pend_tcp->reuse);
 966         comm_point_start_listening(pend_tcp->c, -1, REUSE_TIMEOUT);
 967 }
 968
 969 /** set timeout on tcp fd and setup read event to catch incoming dns msgs */
 970 static void
 971 reuse_tcp_setup_read_and_timeout(struct pending_tcp* pend_tcp)
 972 {
 973         log_reuse_tcp(VERB_CLIENT, "reuse_tcp_setup_readtimeout", &pend_tcp->reuse);
 974         sldns_buffer_clear(pend_tcp->c->buffer);
 975         pend_tcp->c->tcp_is_reading = 1;
 976         pend_tcp->c->tcp_byte_count = 0;
 977         comm_point_stop_listening(pend_tcp->c);
 978         comm_point_start_listening(pend_tcp->c, -1, REUSE_TIMEOUT);
 979 }
 980
 981 int
 982 outnet_tcp_cb(struct comm_point* c, void* arg, int error,
 983         struct comm_reply *reply_info)
 984 {
 985         struct pending_tcp* pend = (struct pending_tcp*)arg;
 986         struct outside_network* outnet = pend->reuse.outnet;
 987         struct waiting_tcp* w = NULL;
 988         verbose(VERB_ALGO, "outnettcp cb");
 989         if(error == NETEVENT_TIMEOUT) {
 990                 if(pend->c->tcp_write_and_read) {
 991                         verbose(VERB_QUERY, "outnettcp got tcp timeout "
 992                                 "for read, ignored because write underway");
 993                         /* if we are writing, ignore readtimer, wait for write timer
 994                          * or write is done */
 995                         return 0;
 996                 } else {
 997                         verbose(VERB_QUERY, "outnettcp got tcp timeout %s",
 998                                 (pend->reuse.tree_by_id.count?"for reading pkt":
 999                                 "for keepalive for reuse"));
1000                 }
1001                 /* must be timeout for reading or keepalive reuse,
1002                  * close it. */
1003                 reuse_tcp_remove_tree_list(outnet, &pend->reuse);
1004         } else if(error == NETEVENT_PKT_WRITTEN) {
1005                 /* the packet we want to write has been written. */
1006                 verbose(VERB_ALGO, "outnet tcp pkt was written event");
1007                 log_assert(c == pend->c);
1008                 log_assert(pend->query->pkt == pend->c->tcp_write_pkt);
1009                 log_assert(pend->query->pkt_len == pend->c->tcp_write_pkt_len);
1010                 pend->c->tcp_write_pkt = NULL;
1011                 pend->c->tcp_write_pkt_len = 0;
1012                 /* the pend.query is already in tree_by_id */
1013                 log_assert(pend->query->id_node.key);
1014                 pend->query = NULL;
1015                 /* setup to write next packet or setup read timeout */
1016                 if(pend->reuse.write_wait_first) {
1017                         verbose(VERB_ALGO, "outnet tcp setup next pkt");
1018                         /* we can write it straight away perhaps, set flag
1019                          * because this callback called after a tcp write
1020                          * succeeded and likely more buffer space is available
1021                          * and we can write some more. */
1022                         pend->reuse.cp_more_write_again = 1;
1023                         pend->query = reuse_write_wait_pop(&pend->reuse);
1024                         comm_point_stop_listening(pend->c);
1025                         outnet_tcp_take_query_setup(pend->c->fd, pend,
1026                                 pend->query);
1027                 } else {
1028                         verbose(VERB_ALGO, "outnet tcp writes done, wait");
1029                         pend->c->tcp_write_and_read = 0;
1030                         pend->reuse.cp_more_read_again = 0;
1031                         pend->reuse.cp_more_write_again = 0;
1032                         pend->c->tcp_is_reading = 1;
1033                         comm_point_stop_listening(pend->c);
1034                         reuse_tcp_setup_timeout(pend);
1035                 }
1036                 return 0;
1037         } else if(error != NETEVENT_NOERROR) {
1038                 verbose(VERB_QUERY, "outnettcp got tcp error %d", error);
1039                 reuse_move_writewait_away(outnet, pend);
1040                 /* pass error below and exit */
1041         } else {
1042                 /* check ID */
1043                 if(sldns_buffer_limit(c->buffer) < sizeof(uint16_t)) {
1044                         log_addr(VERB_QUERY,
1045                                 "outnettcp: bad ID in reply, too short, from:",
1046                                 &pend->reuse.addr, pend->reuse.addrlen);
1047                         error = NETEVENT_CLOSED;
1048                 } else {
1049                         uint16_t id = LDNS_ID_WIRE(sldns_buffer_begin(
1050                                 c->buffer));
1051                         /* find the query the reply is for */
1052                         w = reuse_tcp_by_id_find(&pend->reuse, id);
1053                 }
1054         }
1055         if(error == NETEVENT_NOERROR && !w) {
1056                 /* no struct waiting found in tree, no reply to call */
1057                 log_addr(VERB_QUERY, "outnettcp: bad ID in reply, from:",
1058                         &pend->reuse.addr, pend->reuse.addrlen);
1059                 error = NETEVENT_CLOSED;
1060         }
1061         if(error == NETEVENT_NOERROR) {
1062                 /* add to reuse tree so it can be reused, if not a failure.
1063                  * This is possible if the state machine wants to make a tcp
1064                  * query again to the same destination. */
1065                 if(outnet->tcp_reuse.count < outnet->tcp_reuse_max) {
1066                         (void)reuse_tcp_insert(outnet, pend);
1067                 }
1068         }
1069         if(w) {
1070                 reuse_tree_by_id_delete(&pend->reuse, w);
1071                 verbose(VERB_CLIENT, "outnet tcp callback query err %d buflen %d",
1072                         error, (int)sldns_buffer_limit(c->buffer));
1073                 waiting_tcp_callback(w, c, error, reply_info);
1074                 waiting_tcp_delete(w);
1075         }
1076         verbose(VERB_CLIENT, "outnet_tcp_cb reuse after cb");
1077         if(error == NETEVENT_NOERROR && pend->reuse.node.key) {
1078                 verbose(VERB_CLIENT, "outnet_tcp_cb reuse after cb: keep it");
1079                 /* it is in the reuse_tcp tree, with other queries, or
1080                  * on the empty list. do not decommission it */
1081                 /* if there are more outstanding queries, we could try to
1082                  * read again, to see if it is on the input,
1083                  * because this callback called after a successful read
1084                  * and there could be more bytes to read on the input */
1085                 if(pend->reuse.tree_by_id.count != 0)
1086                         pend->reuse.cp_more_read_again = 1;
1087                 reuse_tcp_setup_read_and_timeout(pend);
1088                 return 0;
1089         }
1090         verbose(VERB_CLIENT, "outnet_tcp_cb reuse after cb: decommission it");
1091         /* no queries on it, no space to keep it. or timeout or closed due
1092          * to error.  Close it */
1093         reuse_cb_and_decommission(outnet, pend, (error==NETEVENT_TIMEOUT?
1094                 NETEVENT_TIMEOUT:NETEVENT_CLOSED));
1095         use_free_buffer(outnet);
1096         return 0;
1097 }
1098
1099 /** lower use count on pc, see if it can be closed */
1100 static void
1101 portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc)
1102 {
1103         struct port_if* pif;
1104         pc->num_outstanding--;
1105         if(pc->num_outstanding > 0) {
1106                 return;
1107         }
1108         /* close it and replace in unused list */
1109         verbose(VERB_ALGO, "close of port %d", pc->number);
1110         comm_point_close(pc->cp);
1111         pif = pc->pif;
1112         log_assert(pif->inuse > 0);
1113 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1114         pif->avail_ports[pif->avail_total - pif->inuse] = pc->number;
1115 #endif
1116         pif->inuse--;
1117         pif->out[pc->index] = pif->out[pif->inuse];
1118         pif->out[pc->index]->index = pc->index;
1119         pc->next = outnet->unused_fds;
1120         outnet->unused_fds = pc;
1121 }
1122
1123 /** try to send waiting UDP queries */
1124 static void
1125 outnet_send_wait_udp(struct outside_network* outnet)
1126 {
1127         struct pending* pend;
1128         /* process waiting queries */
1129         while(outnet->udp_wait_first && outnet->unused_fds
1130                 && !outnet->want_to_quit) {
1131                 pend = outnet->udp_wait_first;
1132                 outnet->udp_wait_first = pend->next_waiting;
1133                 if(!pend->next_waiting) outnet->udp_wait_last = NULL;
1134                 sldns_buffer_clear(outnet->udp_buff);
1135                 sldns_buffer_write(outnet->udp_buff, pend->pkt, pend->pkt_len);
1136                 sldns_buffer_flip(outnet->udp_buff);
1137                 free(pend->pkt); /* freeing now makes get_mem correct */
1138                 pend->pkt = NULL;
1139                 pend->pkt_len = 0;
1140                 if(!randomize_and_send_udp(pend, outnet->udp_buff,
1141                         pend->timeout)) {
1142                         /* callback error on pending */
1143                         if(pend->cb) {
1144                                 fptr_ok(fptr_whitelist_pending_udp(pend->cb));
1145                                 (void)(*pend->cb)(outnet->unused_fds->cp, pend->cb_arg,
1146                                         NETEVENT_CLOSED, NULL);
1147                         }
1148                         pending_delete(outnet, pend);
1149                 }
1150         }
1151 }
1152
1153 int
1154 outnet_udp_cb(struct comm_point* c, void* arg, int error,
1155         struct comm_reply *reply_info)
1156 {
1157         struct outside_network* outnet = (struct outside_network*)arg;
1158         struct pending key;
1159         struct pending* p;
1160         verbose(VERB_ALGO, "answer cb");
1161
1162         if(error != NETEVENT_NOERROR) {
1163                 verbose(VERB_QUERY, "outnetudp got udp error %d", error);
1164                 return 0;
1165         }
1166         if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
1167                 verbose(VERB_QUERY, "outnetudp udp too short");
1168                 return 0;
1169         }
1170         log_assert(reply_info);
1171
1172         /* setup lookup key */
1173         key.id = (unsigned)LDNS_ID_WIRE(sldns_buffer_begin(c->buffer));
1174         memcpy(&key.addr, &reply_info->addr, reply_info->addrlen);
1175         key.addrlen = reply_info->addrlen;
1176         verbose(VERB_ALGO, "Incoming reply id = %4.4x", key.id);
1177         log_addr(VERB_ALGO, "Incoming reply addr =",
1178                 &reply_info->addr, reply_info->addrlen);
1179
1180         /* find it, see if this thing is a valid query response */
1181         verbose(VERB_ALGO, "lookup size is %d entries", (int)outnet->pending->count);
1182         p = (struct pending*)rbtree_search(outnet->pending, &key);
1183         if(!p) {
1184                 verbose(VERB_QUERY, "received unwanted or unsolicited udp reply dropped.");
1185                 log_buf(VERB_ALGO, "dropped message", c->buffer);
1186                 outnet->unwanted_replies++;
1187                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
1188                         >= outnet->unwanted_threshold) {
1189                         log_warn("unwanted reply total reached threshold (%u)"
1190                                 " you may be under attack."
1191                                 " defensive action: clearing the cache",
1192                                 (unsigned)outnet->unwanted_threshold);
1193                         fptr_ok(fptr_whitelist_alloc_cleanup(
1194                                 outnet->unwanted_action));
1195                         (*outnet->unwanted_action)(outnet->unwanted_param);
1196                         outnet->unwanted_total = 0;
1197                 }
1198                 return 0;
1199         }
1200
1201         verbose(VERB_ALGO, "received udp reply.");
1202         log_buf(VERB_ALGO, "udp message", c->buffer);
1203         if(p->pc->cp != c) {
1204                 verbose(VERB_QUERY, "received reply id,addr on wrong port. "
1205                         "dropped.");
1206                 outnet->unwanted_replies++;
1207                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
1208                         >= outnet->unwanted_threshold) {
1209                         log_warn("unwanted reply total reached threshold (%u)"
1210                                 " you may be under attack."
1211                                 " defensive action: clearing the cache",
1212                                 (unsigned)outnet->unwanted_threshold);
1213                         fptr_ok(fptr_whitelist_alloc_cleanup(
1214                                 outnet->unwanted_action));
1215                         (*outnet->unwanted_action)(outnet->unwanted_param);
1216                         outnet->unwanted_total = 0;
1217                 }
1218                 return 0;
1219         }
1220         comm_timer_disable(p->timer);
1221         verbose(VERB_ALGO, "outnet handle udp reply");
1222         /* delete from tree first in case callback creates a retry */
1223         (void)rbtree_delete(outnet->pending, p->node.key);
1224         if(p->cb) {
1225                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
1226                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_NOERROR, reply_info);
1227         }
1228         portcomm_loweruse(outnet, p->pc);
1229         pending_delete(NULL, p);
1230         outnet_send_wait_udp(outnet);
1231         return 0;
1232 }
1233
1234 /** calculate number of ip4 and ip6 interfaces*/
1235 static void
1236 calc_num46(char** ifs, int num_ifs, int do_ip4, int do_ip6,
1237         int* num_ip4, int* num_ip6)
1238 {
1239         int i;
1240         *num_ip4 = 0;
1241         *num_ip6 = 0;
1242         if(num_ifs <= 0) {
1243                 if(do_ip4)
1244                         *num_ip4 = 1;
1245                 if(do_ip6)
1246                         *num_ip6 = 1;
1247                 return;
1248         }
1249         for(i=0; i<num_ifs; i++)
1250         {
1251                 if(str_is_ip6(ifs[i])) {
1252                         if(do_ip6)
1253                                 (*num_ip6)++;
1254                 } else {
1255                         if(do_ip4)
1256                                 (*num_ip4)++;
1257                 }
1258         }
1259
1260 }
1261
1262 void
1263 pending_udp_timer_delay_cb(void* arg)
1264 {
1265         struct pending* p = (struct pending*)arg;
1266         struct outside_network* outnet = p->outnet;
1267         verbose(VERB_ALGO, "timeout udp with delay");
1268         portcomm_loweruse(outnet, p->pc);
1269         pending_delete(outnet, p);
1270         outnet_send_wait_udp(outnet);
1271 }
1272
1273 void
1274 pending_udp_timer_cb(void *arg)
1275 {
1276         struct pending* p = (struct pending*)arg;
1277         struct outside_network* outnet = p->outnet;
1278         /* it timed out */
1279         verbose(VERB_ALGO, "timeout udp");
1280         if(p->cb) {
1281                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
1282                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_TIMEOUT, NULL);
1283         }
1284         /* if delayclose, keep port open for a longer time.
1285          * But if the udpwaitlist exists, then we are struggling to
1286          * keep up with demand for sockets, so do not wait, but service
1287          * the customer (customer service more important than portICMPs) */
1288         if(outnet->delayclose && !outnet->udp_wait_first) {
1289                 p->cb = NULL;
1290                 p->timer->callback = &pending_udp_timer_delay_cb;
1291                 comm_timer_set(p->timer, &outnet->delay_tv);
1292                 return;
1293         }
1294         portcomm_loweruse(outnet, p->pc);
1295         pending_delete(outnet, p);
1296         outnet_send_wait_udp(outnet);
1297 }
1298
1299 /** create pending_tcp buffers */
1300 static int
1301 create_pending_tcp(struct outside_network* outnet, size_t bufsize)
1302 {
1303         size_t i;
1304         if(outnet->num_tcp == 0)
1305                 return 1; /* no tcp needed, nothing to do */
1306         if(!(outnet->tcp_conns = (struct pending_tcp **)calloc(
1307                         outnet->num_tcp, sizeof(struct pending_tcp*))))
1308                 return 0;
1309         for(i=0; i<outnet->num_tcp; i++) {
1310                 if(!(outnet->tcp_conns[i] = (struct pending_tcp*)calloc(1,
1311                         sizeof(struct pending_tcp))))
1312                         return 0;
1313                 outnet->tcp_conns[i]->next_free = outnet->tcp_free;
1314                 outnet->tcp_free = outnet->tcp_conns[i];
1315                 outnet->tcp_conns[i]->c = comm_point_create_tcp_out(
1316                         outnet->base, bufsize, outnet_tcp_cb,
1317                         outnet->tcp_conns[i]);
1318                 if(!outnet->tcp_conns[i]->c)
1319                         return 0;
1320         }
1321         return 1;
1322 }
1323
1324 /** setup an outgoing interface, ready address */
1325 static int setup_if(struct port_if* pif, const char* addrstr,
1326         int* avail, int numavail, size_t numfd)
1327 {
1328 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1329         pif->avail_total = numavail;
1330         pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int));
1331         if(!pif->avail_ports)
1332                 return 0;
1333 #endif
1334         if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) &&
1335            !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT,
1336                               &pif->addr, &pif->addrlen, &pif->pfxlen))
1337                 return 0;
1338         pif->maxout = (int)numfd;
1339         pif->inuse = 0;
1340         pif->out = (struct port_comm**)calloc(numfd,
1341                 sizeof(struct port_comm*));
1342         if(!pif->out)
1343                 return 0;
1344         return 1;
1345 }
1346
1347 struct outside_network*
1348 outside_network_create(struct comm_base *base, size_t bufsize,
1349         size_t num_ports, char** ifs, int num_ifs, int do_ip4,
1350         int do_ip6, size_t num_tcp, int dscp, struct infra_cache* infra,
1351         struct ub_randstate* rnd, int use_caps_for_id, int* availports,
1352         int numavailports, size_t unwanted_threshold, int tcp_mss,
1353         void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
1354         void* sslctx, int delayclose, int tls_use_sni, struct dt_env* dtenv,
1355         int udp_connect)
1356 {
1357         struct outside_network* outnet = (struct outside_network*)
1358                 calloc(1, sizeof(struct outside_network));
1359         size_t k;
1360         if(!outnet) {
1361                 log_err("malloc failed");
1362                 return NULL;
1363         }
1364         comm_base_timept(base, &outnet->now_secs, &outnet->now_tv);
1365         outnet->base = base;
1366         outnet->num_tcp = num_tcp;
1367         outnet->num_tcp_outgoing = 0;
1368         outnet->infra = infra;
1369         outnet->rnd = rnd;
1370         outnet->sslctx = sslctx;
1371         outnet->tls_use_sni = tls_use_sni;
1372 #ifdef USE_DNSTAP
1373         outnet->dtenv = dtenv;
1374 #else
1375         (void)dtenv;
1376 #endif
1377         outnet->svcd_overhead = 0;
1378         outnet->want_to_quit = 0;
1379         outnet->unwanted_threshold = unwanted_threshold;
1380         outnet->unwanted_action = unwanted_action;
1381         outnet->unwanted_param = unwanted_param;
1382         outnet->use_caps_for_id = use_caps_for_id;
1383         outnet->do_udp = do_udp;
1384         outnet->tcp_mss = tcp_mss;
1385         outnet->ip_dscp = dscp;
1386 #ifndef S_SPLINT_S
1387         if(delayclose) {
1388                 outnet->delayclose = 1;
1389                 outnet->delay_tv.tv_sec = delayclose/1000;
1390                 outnet->delay_tv.tv_usec = (delayclose%1000)*1000;
1391         }
1392 #endif
1393         if(udp_connect) {
1394                 outnet->udp_connect = 1;
1395         }
1396         if(numavailports == 0 || num_ports == 0) {
1397                 log_err("no outgoing ports available");
1398                 outside_network_delete(outnet);
1399                 return NULL;
1400         }
1401 #ifndef INET6
1402         do_ip6 = 0;
1403 #endif
1404         calc_num46(ifs, num_ifs, do_ip4, do_ip6,
1405                 &outnet->num_ip4, &outnet->num_ip6);
1406         if(outnet->num_ip4 != 0) {
1407                 if(!(outnet->ip4_ifs = (struct port_if*)calloc(
1408                         (size_t)outnet->num_ip4, sizeof(struct port_if)))) {
1409                         log_err("malloc failed");
1410                         outside_network_delete(outnet);
1411                         return NULL;
1412                 }
1413         }
1414         if(outnet->num_ip6 != 0) {
1415                 if(!(outnet->ip6_ifs = (struct port_if*)calloc(
1416                         (size_t)outnet->num_ip6, sizeof(struct port_if)))) {
1417                         log_err("malloc failed");
1418                         outside_network_delete(outnet);
1419                         return NULL;
1420                 }
1421         }
1422         if(     !(outnet->udp_buff = sldns_buffer_new(bufsize)) ||
1423                 !(outnet->pending = rbtree_create(pending_cmp)) ||
1424                 !(outnet->serviced = rbtree_create(serviced_cmp)) ||
1425                 !create_pending_tcp(outnet, bufsize)) {
1426                 log_err("malloc failed");
1427                 outside_network_delete(outnet);
1428                 return NULL;
1429         }
1430         rbtree_init(&outnet->tcp_reuse, reuse_cmp);
1431         outnet->tcp_reuse_max = num_tcp;
1432
1433         /* allocate commpoints */
1434         for(k=0; k<num_ports; k++) {
1435                 struct port_comm* pc;
1436                 pc = (struct port_comm*)calloc(1, sizeof(*pc));
1437                 if(!pc) {
1438                         log_err("malloc failed");
1439                         outside_network_delete(outnet);
1440                         return NULL;
1441                 }
1442                 pc->cp = comm_point_create_udp(outnet->base, -1,
1443                         outnet->udp_buff, outnet_udp_cb, outnet);
1444                 if(!pc->cp) {
1445                         log_err("malloc failed");
1446                         free(pc);
1447                         outside_network_delete(outnet);
1448                         return NULL;
1449                 }
1450                 pc->next = outnet->unused_fds;
1451                 outnet->unused_fds = pc;
1452         }
1453
1454         /* allocate interfaces */
1455         if(num_ifs == 0) {
1456                 if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0",
1457                         availports, numavailports, num_ports)) {
1458                         log_err("malloc failed");
1459                         outside_network_delete(outnet);
1460                         return NULL;
1461                 }
1462                 if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::",
1463                         availports, numavailports, num_ports)) {
1464                         log_err("malloc failed");
1465                         outside_network_delete(outnet);
1466                         return NULL;
1467                 }
1468         } else {
1469                 size_t done_4 = 0, done_6 = 0;
1470                 int i;
1471                 for(i=0; i<num_ifs; i++) {
1472                         if(str_is_ip6(ifs[i]) && do_ip6) {
1473                                 if(!setup_if(&outnet->ip6_ifs[done_6], ifs[i],
1474                                         availports, numavailports, num_ports)){
1475                                         log_err("malloc failed");
1476                                         outside_network_delete(outnet);
1477                                         return NULL;
1478                                 }
1479                                 done_6++;
1480                         }
1481                         if(!str_is_ip6(ifs[i]) && do_ip4) {
1482                                 if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i],
1483                                         availports, numavailports, num_ports)){
1484                                         log_err("malloc failed");
1485                                         outside_network_delete(outnet);
1486                                         return NULL;
1487                                 }
1488                                 done_4++;
1489                         }
1490                 }
1491         }
1492         return outnet;
1493 }
1494
1495 /** helper pending delete */
1496 static void
1497 pending_node_del(rbnode_type* node, void* arg)
1498 {
1499         struct pending* pend = (struct pending*)node;
1500         struct outside_network* outnet = (struct outside_network*)arg;
1501         pending_delete(outnet, pend);
1502 }
1503
1504 /** helper serviced delete */
1505 static void
1506 serviced_node_del(rbnode_type* node, void* ATTR_UNUSED(arg))
1507 {
1508         struct serviced_query* sq = (struct serviced_query*)node;
1509         struct service_callback* p = sq->cblist, *np;
1510         free(sq->qbuf);
1511         free(sq->zone);
1512         free(sq->tls_auth_name);
1513         edns_opt_list_free(sq->opt_list);
1514         while(p) {
1515                 np = p->next;
1516                 free(p);
1517                 p = np;
1518         }
1519         free(sq);
1520 }
1521
1522 void
1523 outside_network_quit_prepare(struct outside_network* outnet)
1524 {
1525         if(!outnet)
1526                 return;
1527         /* prevent queued items from being sent */
1528         outnet->want_to_quit = 1;
1529 }
1530
1531 void
1532 outside_network_delete(struct outside_network* outnet)
1533 {
1534         if(!outnet)
1535                 return;
1536         outnet->want_to_quit = 1;
1537         /* check every element, since we can be called on malloc error */
1538         if(outnet->pending) {
1539                 /* free pending elements, but do no unlink from tree. */
1540                 traverse_postorder(outnet->pending, pending_node_del, NULL);
1541                 free(outnet->pending);
1542         }
1543         if(outnet->serviced) {
1544                 traverse_postorder(outnet->serviced, serviced_node_del, NULL);
1545                 free(outnet->serviced);
1546         }
1547         if(outnet->udp_buff)
1548                 sldns_buffer_free(outnet->udp_buff);
1549         if(outnet->unused_fds) {
1550                 struct port_comm* p = outnet->unused_fds, *np;
1551                 while(p) {
1552                         np = p->next;
1553                         comm_point_delete(p->cp);
1554                         free(p);
1555                         p = np;
1556                 }
1557                 outnet->unused_fds = NULL;
1558         }
1559         if(outnet->ip4_ifs) {
1560                 int i, k;
1561                 for(i=0; i<outnet->num_ip4; i++) {
1562                         for(k=0; k<outnet->ip4_ifs[i].inuse; k++) {
1563                                 struct port_comm* pc = outnet->ip4_ifs[i].
1564                                         out[k];
1565                                 comm_point_delete(pc->cp);
1566                                 free(pc);
1567                         }
1568 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1569                         free(outnet->ip4_ifs[i].avail_ports);
1570 #endif
1571                         free(outnet->ip4_ifs[i].out);
1572                 }
1573                 free(outnet->ip4_ifs);
1574         }
1575         if(outnet->ip6_ifs) {
1576                 int i, k;
1577                 for(i=0; i<outnet->num_ip6; i++) {
1578                         for(k=0; k<outnet->ip6_ifs[i].inuse; k++) {
1579                                 struct port_comm* pc = outnet->ip6_ifs[i].
1580                                         out[k];
1581                                 comm_point_delete(pc->cp);
1582                                 free(pc);
1583                         }
1584 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1585                         free(outnet->ip6_ifs[i].avail_ports);
1586 #endif
1587                         free(outnet->ip6_ifs[i].out);
1588                 }
1589                 free(outnet->ip6_ifs);
1590         }
1591         if(outnet->tcp_conns) {
1592                 size_t i;
1593                 for(i=0; i<outnet->num_tcp; i++)
1594                         if(outnet->tcp_conns[i]) {
1595                                 if(outnet->tcp_conns[i]->query &&
1596                                         !outnet->tcp_conns[i]->query->
1597                                         on_tcp_waiting_list) {
1598                                         /* delete waiting_tcp elements that
1599                                          * the tcp conn is working on */
1600                                         struct pending_tcp* pend =
1601                                                 (struct pending_tcp*)outnet->
1602                                                 tcp_conns[i]->query->
1603                                                 next_waiting;
1604                                         decommission_pending_tcp(outnet, pend);
1605                                 }
1606                                 comm_point_delete(outnet->tcp_conns[i]->c);
1607                                 waiting_tcp_delete(outnet->tcp_conns[i]->query);
1608                                 free(outnet->tcp_conns[i]);
1609                         }
1610                 free(outnet->tcp_conns);
1611         }
1612         if(outnet->tcp_wait_first) {
1613                 struct waiting_tcp* p = outnet->tcp_wait_first, *np;
1614                 while(p) {
1615                         np = p->next_waiting;
1616                         waiting_tcp_delete(p);
1617                         p = np;
1618                 }
1619         }
1620         /* was allocated in struct pending that was deleted above */
1621         rbtree_init(&outnet->tcp_reuse, reuse_cmp);
1622         outnet->tcp_reuse_first = NULL;
1623         outnet->tcp_reuse_last = NULL;
1624         if(outnet->udp_wait_first) {
1625                 struct pending* p = outnet->udp_wait_first, *np;
1626                 while(p) {
1627                         np = p->next_waiting;
1628                         pending_delete(NULL, p);
1629                         p = np;
1630                 }
1631         }
1632         free(outnet);
1633 }
1634
1635 void
1636 pending_delete(struct outside_network* outnet, struct pending* p)
1637 {
1638         if(!p)
1639                 return;
1640         if(outnet && outnet->udp_wait_first &&
1641                 (p->next_waiting || p == outnet->udp_wait_last) ) {
1642                 /* delete from waiting list, if it is in the waiting list */
1643                 struct pending* prev = NULL, *x = outnet->udp_wait_first;
1644                 while(x && x != p) {
1645                         prev = x;
1646                         x = x->next_waiting;
1647                 }
1648                 if(x) {
1649                         log_assert(x == p);
1650                         if(prev)
1651                                 prev->next_waiting = p->next_waiting;
1652                         else    outnet->udp_wait_first = p->next_waiting;
1653                         if(outnet->udp_wait_last == p)
1654                                 outnet->udp_wait_last = prev;
1655                 }
1656         }
1657         if(outnet) {
1658                 (void)rbtree_delete(outnet->pending, p->node.key);
1659         }
1660         if(p->timer)
1661                 comm_timer_delete(p->timer);
1662         free(p->pkt);
1663         free(p);
1664 }
1665
1666 static void
1667 sai6_putrandom(struct sockaddr_in6 *sa, int pfxlen, struct ub_randstate *rnd)
1668 {
1669         int i, last;
1670         if(!(pfxlen > 0 && pfxlen < 128))
1671                 return;
1672         for(i = 0; i < (128 - pfxlen) / 8; i++) {
1673                 sa->sin6_addr.s6_addr[15-i] = (uint8_t)ub_random_max(rnd, 256);
1674         }
1675         last = pfxlen & 7;
1676         if(last != 0) {
1677                 sa->sin6_addr.s6_addr[15-i] |=
1678                         ((0xFF >> last) & ub_random_max(rnd, 256));
1679         }
1680 }
1681
1682 /**
1683  * Try to open a UDP socket for outgoing communication.
1684  * Sets sockets options as needed.
1685  * @param addr: socket address.
1686  * @param addrlen: length of address.
1687  * @param pfxlen: length of network prefix (for address randomisation).
1688  * @param port: port override for addr.
1689  * @param inuse: if -1 is returned, this bool means the port was in use.
1690  * @param rnd: random state (for address randomisation).
1691  * @param dscp: DSCP to use.
1692  * @return fd or -1
1693  */
1694 static int
1695 udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen,
1696         int port, int* inuse, struct ub_randstate* rnd, int dscp)
1697 {
1698         int fd, noproto;
1699         if(addr_is_ip6(addr, addrlen)) {
1700                 int freebind = 0;
1701                 struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
1702                 sa.sin6_port = (in_port_t)htons((uint16_t)port);
1703                 sa.sin6_flowinfo = 0;
1704                 sa.sin6_scope_id = 0;
1705                 if(pfxlen != 0) {
1706                         freebind = 1;
1707                         sai6_putrandom(&sa, pfxlen, rnd);
1708                 }
1709                 fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
1710                         (struct sockaddr*)&sa, addrlen, 1, inuse, &noproto,
1711                         0, 0, 0, NULL, 0, freebind, 0, dscp);
1712         } else {
1713                 struct sockaddr_in* sa = (struct sockaddr_in*)addr;
1714                 sa->sin_port = (in_port_t)htons((uint16_t)port);
1715                 fd = create_udp_sock(AF_INET, SOCK_DGRAM,
1716                         (struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
1717                         0, 0, 0, NULL, 0, 0, 0, dscp);
1718         }
1719         return fd;
1720 }
1721
1722 /** Select random ID */
1723 static int
1724 select_id(struct outside_network* outnet, struct pending* pend,
1725         sldns_buffer* packet)
1726 {
1727         int id_tries = 0;
1728         pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
1729         LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
1730
1731         /* insert in tree */
1732         pend->node.key = pend;
1733         while(!rbtree_insert(outnet->pending, &pend->node)) {
1734                 /* change ID to avoid collision */
1735                 pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
1736                 LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
1737                 id_tries++;
1738                 if(id_tries == MAX_ID_RETRY) {
1739                         pend->id=99999; /* non existant ID */
1740                         log_err("failed to generate unique ID, drop msg");
1741                         return 0;
1742                 }
1743         }
1744         verbose(VERB_ALGO, "inserted new pending reply id=%4.4x", pend->id);
1745         return 1;
1746 }
1747
1748 /** Select random interface and port */
1749 static int
1750 select_ifport(struct outside_network* outnet, struct pending* pend,
1751         int num_if, struct port_if* ifs)
1752 {
1753         int my_if, my_port, fd, portno, inuse, tries=0;
1754         struct port_if* pif;
1755         /* randomly select interface and port */
1756         if(num_if == 0) {
1757                 verbose(VERB_QUERY, "Need to send query but have no "
1758                         "outgoing interfaces of that family");
1759                 return 0;
1760         }
1761         log_assert(outnet->unused_fds);
1762         tries = 0;
1763         while(1) {
1764                 my_if = ub_random_max(outnet->rnd, num_if);
1765                 pif = &ifs[my_if];
1766 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1767                 if(outnet->udp_connect) {
1768                         /* if we connect() we cannot reuse fds for a port */
1769                         if(pif->inuse >= pif->avail_total) {
1770                                 tries++;
1771                                 if(tries < MAX_PORT_RETRY)
1772                                         continue;
1773                                 log_err("failed to find an open port, drop msg");
1774                                 return 0;
1775                         }
1776                         my_port = pif->inuse + ub_random_max(outnet->rnd,
1777                                 pif->avail_total - pif->inuse);
1778                 } else  {
1779                         my_port = ub_random_max(outnet->rnd, pif->avail_total);
1780                         if(my_port < pif->inuse) {
1781                                 /* port already open */
1782                                 pend->pc = pif->out[my_port];
1783                                 verbose(VERB_ALGO, "using UDP if=%d port=%d",
1784                                         my_if, pend->pc->number);
1785                                 break;
1786                         }
1787                 }
1788                 /* try to open new port, if fails, loop to try again */
1789                 log_assert(pif->inuse < pif->maxout);
1790                 portno = pif->avail_ports[my_port - pif->inuse];
1791 #else
1792                 my_port = portno = 0;
1793 #endif
1794                 fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen,
1795                         portno, &inuse, outnet->rnd, outnet->ip_dscp);
1796                 if(fd == -1 && !inuse) {
1797                         /* nonrecoverable error making socket */
1798                         return 0;
1799                 }
1800                 if(fd != -1) {
1801                         verbose(VERB_ALGO, "opened UDP if=%d port=%d",
1802                                 my_if, portno);
1803                         if(outnet->udp_connect) {
1804                                 /* connect() to the destination */
1805                                 if(connect(fd, (struct sockaddr*)&pend->addr,
1806                                         pend->addrlen) < 0) {
1807                                         log_err_addr("udp connect failed",
1808                                                 strerror(errno), &pend->addr,
1809                                                 pend->addrlen);
1810                                         sock_close(fd);
1811                                         return 0;
1812                                 }
1813                         }
1814                         /* grab fd */
1815                         pend->pc = outnet->unused_fds;
1816                         outnet->unused_fds = pend->pc->next;
1817
1818                         /* setup portcomm */
1819                         pend->pc->next = NULL;
1820                         pend->pc->number = portno;
1821                         pend->pc->pif = pif;
1822                         pend->pc->index = pif->inuse;
1823                         pend->pc->num_outstanding = 0;
1824                         comm_point_start_listening(pend->pc->cp, fd, -1);
1825
1826                         /* grab port in interface */
1827                         pif->out[pif->inuse] = pend->pc;
1828 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1829                         pif->avail_ports[my_port - pif->inuse] =
1830                                 pif->avail_ports[pif->avail_total-pif->inuse-1];
1831 #endif
1832                         pif->inuse++;
1833                         break;
1834                 }
1835                 /* failed, already in use */
1836                 verbose(VERB_QUERY, "port %d in use, trying another", portno);
1837                 tries++;
1838                 if(tries == MAX_PORT_RETRY) {
1839                         log_err("failed to find an open port, drop msg");
1840                         return 0;
1841                 }
1842         }
1843         log_assert(pend->pc);
1844         pend->pc->num_outstanding++;
1845
1846         return 1;
1847 }
1848
1849 static int
1850 randomize_and_send_udp(struct pending* pend, sldns_buffer* packet, int timeout)
1851 {
1852         struct timeval tv;
1853         struct outside_network* outnet = pend->sq->outnet;
1854
1855         /* select id */
1856         if(!select_id(outnet, pend, packet)) {
1857                 return 0;
1858         }
1859
1860         /* select src_if, port */
1861         if(addr_is_ip6(&pend->addr, pend->addrlen)) {
1862                 if(!select_ifport(outnet, pend,
1863                         outnet->num_ip6, outnet->ip6_ifs))
1864                         return 0;
1865         } else {
1866                 if(!select_ifport(outnet, pend,
1867                         outnet->num_ip4, outnet->ip4_ifs))
1868                         return 0;
1869         }
1870         log_assert(pend->pc && pend->pc->cp);
1871
1872         /* send it over the commlink */
1873         if(outnet->udp_connect) {
1874                 if(!comm_point_send_udp_msg(pend->pc->cp, packet, NULL, 0)) {
1875                         portcomm_loweruse(outnet, pend->pc);
1876                         return 0;
1877                 }
1878         } else {
1879                 if(!comm_point_send_udp_msg(pend->pc->cp, packet,
1880                         (struct sockaddr*)&pend->addr, pend->addrlen)) {
1881                         portcomm_loweruse(outnet, pend->pc);
1882                         return 0;
1883                 }
1884         }
1885
1886         /* system calls to set timeout after sending UDP to make roundtrip
1887            smaller. */
1888 #ifndef S_SPLINT_S
1889         tv.tv_sec = timeout/1000;
1890         tv.tv_usec = (timeout%1000)*1000;
1891 #endif
1892         comm_timer_set(pend->timer, &tv);
1893
1894 #ifdef USE_DNSTAP
1895         if(outnet->dtenv &&
1896            (outnet->dtenv->log_resolver_query_messages ||
1897             outnet->dtenv->log_forwarder_query_messages))
1898                 dt_msg_send_outside_query(outnet->dtenv, &pend->addr, comm_udp,
1899                 pend->sq->zone, pend->sq->zonelen, packet);
1900 #endif
1901         return 1;
1902 }
1903
1904 struct pending*
1905 pending_udp_query(struct serviced_query* sq, struct sldns_buffer* packet,
1906         int timeout, comm_point_callback_type* cb, void* cb_arg)
1907 {
1908         struct pending* pend = (struct pending*)calloc(1, sizeof(*pend));
1909         if(!pend) return NULL;
1910         pend->outnet = sq->outnet;
1911         pend->sq = sq;
1912         pend->addrlen = sq->addrlen;
1913         memmove(&pend->addr, &sq->addr, sq->addrlen);
1914         pend->cb = cb;
1915         pend->cb_arg = cb_arg;
1916         pend->node.key = pend;
1917         pend->timer = comm_timer_create(sq->outnet->base, pending_udp_timer_cb,
1918                 pend);
1919         if(!pend->timer) {
1920                 free(pend);
1921                 return NULL;
1922         }
1923
1924         if(sq->outnet->unused_fds == NULL) {
1925                 /* no unused fd, cannot create a new port (randomly) */
1926                 verbose(VERB_ALGO, "no fds available, udp query waiting");
1927                 pend->timeout = timeout;
1928                 pend->pkt_len = sldns_buffer_limit(packet);
1929                 pend->pkt = (uint8_t*)memdup(sldns_buffer_begin(packet),
1930                         pend->pkt_len);
1931                 if(!pend->pkt) {
1932                         comm_timer_delete(pend->timer);
1933                         free(pend);
1934                         return NULL;
1935                 }
1936                 /* put at end of waiting list */
1937                 if(sq->outnet->udp_wait_last)
1938                         sq->outnet->udp_wait_last->next_waiting = pend;
1939                 else
1940                         sq->outnet->udp_wait_first = pend;
1941                 sq->outnet->udp_wait_last = pend;
1942                 return pend;
1943         }
1944         if(!randomize_and_send_udp(pend, packet, timeout)) {
1945                 pending_delete(sq->outnet, pend);
1946                 return NULL;
1947         }
1948         return pend;
1949 }
1950
1951 void
1952 outnet_tcptimer(void* arg)
1953 {
1954         struct waiting_tcp* w = (struct waiting_tcp*)arg;
1955         struct outside_network* outnet = w->outnet;
1956         verbose(VERB_CLIENT, "outnet_tcptimer");
1957         if(w->on_tcp_waiting_list) {
1958                 /* it is on the waiting list */
1959                 waiting_list_remove(outnet, w);
1960                 waiting_tcp_callback(w, NULL, NETEVENT_TIMEOUT, NULL);
1961                 waiting_tcp_delete(w);
1962         } else {
1963                 /* it was in use */
1964                 struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
1965                 reuse_cb_and_decommission(outnet, pend, NETEVENT_TIMEOUT);
1966         }
1967         use_free_buffer(outnet);
1968 }
1969
1970 /** close the oldest reuse_tcp connection to make a fd and struct pend
1971  * available for a new stream connection */
1972 static void
1973 reuse_tcp_close_oldest(struct outside_network* outnet)
1974 {
1975         struct pending_tcp* pend;
1976         verbose(VERB_CLIENT, "reuse_tcp_close_oldest");
1977         if(!outnet->tcp_reuse_last) return;
1978         pend = outnet->tcp_reuse_last->pending;
1979
1980         /* snip off of LRU */
1981         log_assert(pend->reuse.lru_next == NULL);
1982         if(pend->reuse.lru_prev) {
1983                 outnet->tcp_reuse_last = pend->reuse.lru_prev;
1984                 pend->reuse.lru_prev->lru_next = NULL;
1985         } else {
1986                 outnet->tcp_reuse_last = NULL;
1987                 outnet->tcp_reuse_first = NULL;
1988         }
1989         pend->reuse.item_on_lru_list = 0;
1990
1991         /* free up */
1992         reuse_cb_and_decommission(outnet, pend, NETEVENT_CLOSED);
1993 }
1994
1995 /** find spare ID value for reuse tcp stream.  That is random and also does
1996  * not collide with an existing query ID that is in use or waiting */
1997 uint16_t
1998 reuse_tcp_select_id(struct reuse_tcp* reuse, struct outside_network* outnet)
1999 {
2000         uint16_t id = 0, curid, nextid;
2001         const int try_random = 2000;
2002         int i;
2003         unsigned select, count, space;
2004         rbnode_type* node;
2005
2006         /* make really sure the tree is not empty */
2007         if(reuse->tree_by_id.count == 0) {
2008                 id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
2009                 return id;
2010         }
2011
2012         /* try to find random empty spots by picking them */
2013         for(i = 0; i<try_random; i++) {
2014                 id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
2015                 if(!reuse_tcp_by_id_find(reuse, id)) {
2016                         return id;
2017                 }
2018         }
2019
2020         /* equally pick a random unused element from the tree that is
2021          * not in use.  Pick a the n-th index of an ununused number,
2022          * then loop over the empty spaces in the tree and find it */
2023         log_assert(reuse->tree_by_id.count < 0xffff);
2024         select = ub_random_max(outnet->rnd, 0xffff - reuse->tree_by_id.count);
2025         /* select value now in 0 .. num free - 1 */
2026
2027         count = 0; /* number of free spaces passed by */
2028         node = rbtree_first(&reuse->tree_by_id);
2029         log_assert(node && node != RBTREE_NULL); /* tree not empty */
2030         /* see if select is before first node */
2031         if(select < tree_by_id_get_id(node))
2032                 return select;
2033         count += tree_by_id_get_id(node);
2034         /* perhaps select is between nodes */
2035         while(node && node != RBTREE_NULL) {
2036                 rbnode_type* next = rbtree_next(node);
2037                 if(next && next != RBTREE_NULL) {
2038                         curid = tree_by_id_get_id(node);
2039                         nextid = tree_by_id_get_id(next);
2040                         log_assert(curid < nextid);
2041                         if(curid != 0xffff && curid + 1 < nextid) {
2042                                 /* space between nodes */
2043                                 space = nextid - curid - 1;
2044                                 log_assert(select >= count);
2045                                 if(select < count + space) {
2046                                         /* here it is */
2047                                         return curid + 1 + (select - count);
2048                                 }
2049                                 count += space;
2050                         }
2051                 }
2052                 node = next;
2053         }
2054
2055         /* select is after the last node */
2056         /* count is the number of free positions before the nodes in the
2057          * tree */
2058         node = rbtree_last(&reuse->tree_by_id);
2059         log_assert(node && node != RBTREE_NULL); /* tree not empty */
2060         curid = tree_by_id_get_id(node);
2061         log_assert(count + (0xffff-curid) + reuse->tree_by_id.count == 0xffff);
2062         return curid + 1 + (select - count);
2063 }
2064
2065 struct waiting_tcp*
2066 pending_tcp_query(struct serviced_query* sq, sldns_buffer* packet,
2067         int timeout, comm_point_callback_type* callback, void* callback_arg)
2068 {
2069         struct pending_tcp* pend = sq->outnet->tcp_free;
2070         struct reuse_tcp* reuse = NULL;
2071         struct waiting_tcp* w;
2072
2073         verbose(VERB_CLIENT, "pending_tcp_query");
2074         if(sldns_buffer_limit(packet) < sizeof(uint16_t)) {
2075                 verbose(VERB_ALGO, "pending tcp query with too short buffer < 2");
2076                 return NULL;
2077         }
2078
2079         /* find out if a reused stream to the target exists */
2080         /* if so, take it into use */
2081         reuse = reuse_tcp_find(sq->outnet, &sq->addr, sq->addrlen,
2082                 sq->ssl_upstream);
2083         if(reuse) {
2084                 log_reuse_tcp(VERB_CLIENT, "pending_tcp_query: found reuse", reuse);
2085                 log_assert(reuse->pending);
2086                 pend = reuse->pending;
2087                 reuse_tcp_lru_touch(sq->outnet, reuse);
2088         }
2089
2090         /* if !pend but we have reuse streams, close a reuse stream
2091          * to be able to open a new one to this target, no use waiting
2092          * to reuse a file descriptor while another query needs to use
2093          * that buffer and file descriptor now. */
2094         if(!pend) {
2095                 reuse_tcp_close_oldest(sq->outnet);
2096                 pend = sq->outnet->tcp_free;
2097         }
2098
2099         /* allocate space to store query */
2100         w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp)
2101                 + sldns_buffer_limit(packet));
2102         if(!w) {
2103                 return NULL;
2104         }
2105         if(!(w->timer = comm_timer_create(sq->outnet->base, outnet_tcptimer, w))) {
2106                 free(w);
2107                 return NULL;
2108         }
2109         w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
2110         w->pkt_len = sldns_buffer_limit(packet);
2111         memmove(w->pkt, sldns_buffer_begin(packet), w->pkt_len);
2112         if(reuse)
2113                 w->id = reuse_tcp_select_id(reuse, sq->outnet);
2114         else    w->id = ((unsigned)ub_random(sq->outnet->rnd)>>8) & 0xffff;
2115         LDNS_ID_SET(w->pkt, w->id);
2116         memcpy(&w->addr, &sq->addr, sq->addrlen);
2117         w->addrlen = sq->addrlen;
2118         w->outnet = sq->outnet;
2119         w->on_tcp_waiting_list = 0;
2120         w->next_waiting = NULL;
2121         w->cb = callback;
2122         w->cb_arg = callback_arg;
2123         w->ssl_upstream = sq->ssl_upstream;
2124         w->tls_auth_name = sq->tls_auth_name;
2125         w->timeout = timeout;
2126         w->id_node.key = NULL;
2127         w->write_wait_prev = NULL;
2128         w->write_wait_next = NULL;
2129         w->write_wait_queued = 0;
2130         w->error_count = 0;
2131         if(pend) {
2132                 /* we have a buffer available right now */
2133                 if(reuse) {
2134                         /* reuse existing fd, write query and continue */
2135                         /* store query in tree by id */
2136                         verbose(VERB_CLIENT, "pending_tcp_query: reuse, store");
2137                         w->next_waiting = (void*)pend;
2138                         reuse_tree_by_id_insert(&pend->reuse, w);
2139                         /* can we write right now? */
2140                         if(pend->query == NULL) {
2141                                 /* write straight away */
2142                                 /* stop the timer on read of the fd */
2143                                 comm_point_stop_listening(pend->c);
2144                                 pend->query = w;
2145                                 outnet_tcp_take_query_setup(pend->c->fd, pend,
2146                                         w);
2147                         } else {
2148                                 /* put it in the waiting list for
2149                                  * this stream */
2150                                 reuse_write_wait_push_back(&pend->reuse, w);
2151                         }
2152                 } else {
2153                         /* create new fd and connect to addr, setup to
2154                          * write query */
2155                         verbose(VERB_CLIENT, "pending_tcp_query: new fd, connect");
2156                         rbtree_init(&pend->reuse.tree_by_id, reuse_id_cmp);
2157                         pend->reuse.pending = pend;
2158                         memcpy(&pend->reuse.addr, &sq->addr, sq->addrlen);
2159                         pend->reuse.addrlen = sq->addrlen;
2160                         if(!outnet_tcp_take_into_use(w)) {
2161                                 waiting_tcp_delete(w);
2162                                 return NULL;
2163                         }
2164                 }
2165         } else {
2166                 /* queue up */
2167                 /* waiting for a buffer on the outside network buffer wait
2168                  * list */
2169                 verbose(VERB_CLIENT, "pending_tcp_query: queue to wait");
2170                 outnet_add_tcp_waiting(sq->outnet, w);
2171         }
2172 #ifdef USE_DNSTAP
2173         if(sq->outnet->dtenv &&
2174            (sq->outnet->dtenv->log_resolver_query_messages ||
2175             sq->outnet->dtenv->log_forwarder_query_messages))
2176                 dt_msg_send_outside_query(sq->outnet->dtenv, &sq->addr,
2177                         comm_tcp, sq->zone, sq->zonelen, packet);
2178 #endif
2179         return w;
2180 }
2181
2182 /** create query for serviced queries */
2183 static void
2184 serviced_gen_query(sldns_buffer* buff, uint8_t* qname, size_t qnamelen,
2185         uint16_t qtype, uint16_t qclass, uint16_t flags)
2186 {
2187         sldns_buffer_clear(buff);
2188         /* skip id */
2189         sldns_buffer_write_u16(buff, flags);
2190         sldns_buffer_write_u16(buff, 1); /* qdcount */
2191         sldns_buffer_write_u16(buff, 0); /* ancount */
2192         sldns_buffer_write_u16(buff, 0); /* nscount */
2193         sldns_buffer_write_u16(buff, 0); /* arcount */
2194         sldns_buffer_write(buff, qname, qnamelen);
2195         sldns_buffer_write_u16(buff, qtype);
2196         sldns_buffer_write_u16(buff, qclass);
2197         sldns_buffer_flip(buff);
2198 }
2199
2200 /** lookup serviced query in serviced query rbtree */
2201 static struct serviced_query*
2202 lookup_serviced(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
2203         struct sockaddr_storage* addr, socklen_t addrlen,
2204         struct edns_option* opt_list)
2205 {
2206         struct serviced_query key;
2207         key.node.key = &key;
2208         key.qbuf = sldns_buffer_begin(buff);
2209         key.qbuflen = sldns_buffer_limit(buff);
2210         key.dnssec = dnssec;
2211         memcpy(&key.addr, addr, addrlen);
2212         key.addrlen = addrlen;
2213         key.outnet = outnet;
2214         key.opt_list = opt_list;
2215         return (struct serviced_query*)rbtree_search(outnet->serviced, &key);
2216 }
2217
2218 /** Create new serviced entry */
2219 static struct serviced_query*
2220 serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
2221         int want_dnssec, int nocaps, int tcp_upstream, int ssl_upstream,
2222         char* tls_auth_name, struct sockaddr_storage* addr, socklen_t addrlen,
2223         uint8_t* zone, size_t zonelen, int qtype, struct edns_option* opt_list)
2224 {
2225         struct serviced_query* sq = (struct serviced_query*)malloc(sizeof(*sq));
2226 #ifdef UNBOUND_DEBUG
2227         rbnode_type* ins;
2228 #endif
2229         if(!sq)
2230                 return NULL;
2231         sq->node.key = sq;
2232         sq->qbuf = memdup(sldns_buffer_begin(buff), sldns_buffer_limit(buff));
2233         if(!sq->qbuf) {
2234                 free(sq);
2235                 return NULL;
2236         }
2237         sq->qbuflen = sldns_buffer_limit(buff);
2238         sq->zone = memdup(zone, zonelen);
2239         if(!sq->zone) {
2240                 free(sq->qbuf);
2241                 free(sq);
2242                 return NULL;
2243         }
2244         sq->zonelen = zonelen;
2245         sq->qtype = qtype;
2246         sq->dnssec = dnssec;
2247         sq->want_dnssec = want_dnssec;
2248         sq->nocaps = nocaps;
2249         sq->tcp_upstream = tcp_upstream;
2250         sq->ssl_upstream = ssl_upstream;
2251         if(tls_auth_name) {
2252                 sq->tls_auth_name = strdup(tls_auth_name);
2253                 if(!sq->tls_auth_name) {
2254                         free(sq->zone);
2255                         free(sq->qbuf);
2256                         free(sq);
2257                         return NULL;
2258                 }
2259         } else {
2260                 sq->tls_auth_name = NULL;
2261         }
2262         memcpy(&sq->addr, addr, addrlen);
2263         sq->addrlen = addrlen;
2264         sq->opt_list = NULL;
2265         if(opt_list) {
2266                 sq->opt_list = edns_opt_copy_alloc(opt_list);
2267                 if(!sq->opt_list) {
2268                         free(sq->tls_auth_name);
2269                         free(sq->zone);
2270                         free(sq->qbuf);
2271                         free(sq);
2272                         return NULL;
2273                 }
2274         }
2275         sq->outnet = outnet;
2276         sq->cblist = NULL;
2277         sq->pending = NULL;
2278         sq->status = serviced_initial;
2279         sq->retry = 0;
2280         sq->to_be_deleted = 0;
2281 #ifdef UNBOUND_DEBUG
2282         ins =
2283 #else
2284         (void)
2285 #endif
2286         rbtree_insert(outnet->serviced, &sq->node);
2287         log_assert(ins != NULL); /* must not be already present */
2288         return sq;
2289 }
2290
2291 /** remove waiting tcp from the outnet waiting list */
2292 static void
2293 waiting_list_remove(struct outside_network* outnet, struct waiting_tcp* w)
2294 {
2295         struct waiting_tcp* p = outnet->tcp_wait_first, *prev = NULL;
2296         w->on_tcp_waiting_list = 0;
2297         while(p) {
2298                 if(p == w) {
2299                         /* remove w */
2300                         if(prev)
2301                                 prev->next_waiting = w->next_waiting;
2302                         else    outnet->tcp_wait_first = w->next_waiting;
2303                         if(outnet->tcp_wait_last == w)
2304                                 outnet->tcp_wait_last = prev;
2305                         return;
2306                 }
2307                 prev = p;
2308                 p = p->next_waiting;
2309         }
2310 }
2311
2312 /** reuse tcp stream, remove serviced query from stream,
2313  * return true if the stream is kept, false if it is to be closed */
2314 static int
2315 reuse_tcp_remove_serviced_keep(struct waiting_tcp* w,
2316         struct serviced_query* sq)
2317 {
2318         struct pending_tcp* pend_tcp = (struct pending_tcp*)w->next_waiting;
2319         verbose(VERB_CLIENT, "reuse_tcp_remove_serviced_keep");
2320         /* remove the callback. let query continue to write to not cancel
2321          * the stream itself.  also keep it as an entry in the tree_by_id,
2322          * in case the answer returns (that we no longer want), but we cannot
2323          * pick the same ID number meanwhile */
2324         w->cb = NULL;
2325         /* see if can be entered in reuse tree
2326          * for that the FD has to be non-1 */
2327         if(pend_tcp->c->fd == -1) {
2328                 verbose(VERB_CLIENT, "reuse_tcp_remove_serviced_keep: -1 fd");
2329                 return 0;
2330         }
2331         /* if in tree and used by other queries */
2332         if(pend_tcp->reuse.node.key) {
2333                 verbose(VERB_CLIENT, "reuse_tcp_remove_serviced_keep: in use by other queries");
2334                 /* do not reset the keepalive timer, for that
2335                  * we'd need traffic, and this is where the serviced is
2336                  * removed due to state machine internal reasons,
2337                  * eg. iterator no longer interested in this query */
2338                 return 1;
2339         }
2340         /* if still open and want to keep it open */
2341         if(pend_tcp->c->fd != -1 && sq->outnet->tcp_reuse.count <
2342                 sq->outnet->tcp_reuse_max) {
2343                 verbose(VERB_CLIENT, "reuse_tcp_remove_serviced_keep: keep open");
2344                 /* set a keepalive timer on it */
2345                 if(!reuse_tcp_insert(sq->outnet, pend_tcp)) {
2346                         return 0;
2347                 }
2348                 reuse_tcp_setup_timeout(pend_tcp);
2349                 return 1;
2350         }
2351         return 0;
2352 }
2353
2354 /** cleanup serviced query entry */
2355 static void
2356 serviced_delete(struct serviced_query* sq)
2357 {
2358         verbose(VERB_CLIENT, "serviced_delete");
2359         if(sq->pending) {
2360                 /* clear up the pending query */
2361                 if(sq->status == serviced_query_UDP_EDNS ||
2362                         sq->status == serviced_query_UDP ||
2363                         sq->status == serviced_query_UDP_EDNS_FRAG ||
2364                         sq->status == serviced_query_UDP_EDNS_fallback) {
2365                         struct pending* p = (struct pending*)sq->pending;
2366                         verbose(VERB_CLIENT, "serviced_delete: UDP");
2367                         if(p->pc)
2368                                 portcomm_loweruse(sq->outnet, p->pc);
2369                         pending_delete(sq->outnet, p);
2370                         /* this call can cause reentrant calls back into the
2371                          * mesh */
2372                         outnet_send_wait_udp(sq->outnet);
2373                 } else {
2374                         struct waiting_tcp* w = (struct waiting_tcp*)
2375                                 sq->pending;
2376                         verbose(VERB_CLIENT, "serviced_delete: TCP");
2377                         /* if on stream-write-waiting list then
2378                          * remove from waiting list and waiting_tcp_delete */
2379                         if(w->write_wait_queued) {
2380                                 struct pending_tcp* pend =
2381                                         (struct pending_tcp*)w->next_waiting;
2382                                 verbose(VERB_CLIENT, "serviced_delete: writewait");
2383                                 reuse_tree_by_id_delete(&pend->reuse, w);
2384                                 reuse_write_wait_remove(&pend->reuse, w);
2385                                 waiting_tcp_delete(w);
2386                         } else if(!w->on_tcp_waiting_list) {
2387                                 struct pending_tcp* pend =
2388                                         (struct pending_tcp*)w->next_waiting;
2389                                 verbose(VERB_CLIENT, "serviced_delete: tcpreusekeep");
2390                                 if(!reuse_tcp_remove_serviced_keep(w, sq)) {
2391                                         reuse_cb_and_decommission(sq->outnet,
2392                                                 pend, NETEVENT_CLOSED);
2393                                         use_free_buffer(sq->outnet);
2394                                 }
2395                                 sq->pending = NULL;
2396                         } else {
2397                                 verbose(VERB_CLIENT, "serviced_delete: tcpwait");
2398                                 waiting_list_remove(sq->outnet, w);
2399                                 waiting_tcp_delete(w);
2400                         }
2401                 }
2402         }
2403         /* does not delete from tree, caller has to do that */
2404         serviced_node_del(&sq->node, NULL);
2405 }
2406
2407 /** perturb a dname capitalization randomly */
2408 static void
2409 serviced_perturb_qname(struct ub_randstate* rnd, uint8_t* qbuf, size_t len)
2410 {
2411         uint8_t lablen;
2412         uint8_t* d = qbuf + 10;
2413         long int random = 0;
2414         int bits = 0;
2415         log_assert(len >= 10 + 5 /* offset qname, root, qtype, qclass */);
2416         (void)len;
2417         lablen = *d++;
2418         while(lablen) {
2419                 while(lablen--) {
2420                         /* only perturb A-Z, a-z */
2421                         if(isalpha((unsigned char)*d)) {
2422                                 /* get a random bit */
2423                                 if(bits == 0) {
2424                                         random = ub_random(rnd);
2425                                         bits = 30;
2426                                 }
2427                                 if(random & 0x1) {
2428                                         *d = (uint8_t)toupper((unsigned char)*d);
2429                                 } else {
2430                                         *d = (uint8_t)tolower((unsigned char)*d);
2431                                 }
2432                                 random >>= 1;
2433                                 bits--;
2434                         }
2435                         d++;
2436                 }
2437                 lablen = *d++;
2438         }
2439         if(verbosity >= VERB_ALGO) {
2440                 char buf[LDNS_MAX_DOMAINLEN+1];
2441                 dname_str(qbuf+10, buf);
2442                 verbose(VERB_ALGO, "qname perturbed to %s", buf);
2443         }
2444 }
2445
2446 /** put serviced query into a buffer */
2447 static void
2448 serviced_encode(struct serviced_query* sq, sldns_buffer* buff, int with_edns)
2449 {
2450         /* if we are using 0x20 bits for ID randomness, perturb them */
2451         if(sq->outnet->use_caps_for_id && !sq->nocaps) {
2452                 serviced_perturb_qname(sq->outnet->rnd, sq->qbuf, sq->qbuflen);
2453         }
2454         /* generate query */
2455         sldns_buffer_clear(buff);
2456         sldns_buffer_write_u16(buff, 0); /* id placeholder */
2457         sldns_buffer_write(buff, sq->qbuf, sq->qbuflen);
2458         sldns_buffer_flip(buff);
2459         if(with_edns) {
2460                 /* add edns section */
2461                 struct edns_data edns;
2462                 edns.edns_present = 1;
2463                 edns.ext_rcode = 0;
2464                 edns.edns_version = EDNS_ADVERTISED_VERSION;
2465                 edns.opt_list = sq->opt_list;
2466                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
2467                         if(addr_is_ip6(&sq->addr, sq->addrlen)) {
2468                                 if(EDNS_FRAG_SIZE_IP6 < EDNS_ADVERTISED_SIZE)
2469                                         edns.udp_size = EDNS_FRAG_SIZE_IP6;
2470                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
2471                         } else {
2472                                 if(EDNS_FRAG_SIZE_IP4 < EDNS_ADVERTISED_SIZE)
2473                                         edns.udp_size = EDNS_FRAG_SIZE_IP4;
2474                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
2475                         }
2476                 } else {
2477                         edns.udp_size = EDNS_ADVERTISED_SIZE;
2478                 }
2479                 edns.bits = 0;
2480                 if(sq->dnssec & EDNS_DO)
2481                         edns.bits = EDNS_DO;
2482                 if(sq->dnssec & BIT_CD)
2483                         LDNS_CD_SET(sldns_buffer_begin(buff));
2484                 attach_edns_record(buff, &edns);
2485         }
2486 }
2487
2488 /**
2489  * Perform serviced query UDP sending operation.
2490  * Sends UDP with EDNS, unless infra host marked non EDNS.
2491  * @param sq: query to send.
2492  * @param buff: buffer scratch space.
2493  * @return 0 on error.
2494  */
2495 static int
2496 serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
2497 {
2498         int rtt, vs;
2499         uint8_t edns_lame_known;
2500         time_t now = *sq->outnet->now_secs;
2501
2502         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
2503                 sq->zonelen, now, &vs, &edns_lame_known, &rtt))
2504                 return 0;
2505         sq->last_rtt = rtt;
2506         verbose(VERB_ALGO, "EDNS lookup known=%d vs=%d", edns_lame_known, vs);
2507         if(sq->status == serviced_initial) {
2508                 if(vs != -1) {
2509                         sq->status = serviced_query_UDP_EDNS;
2510                 } else {
2511                         sq->status = serviced_query_UDP;
2512                 }
2513         }
2514         serviced_encode(sq, buff, (sq->status == serviced_query_UDP_EDNS) ||
2515                 (sq->status == serviced_query_UDP_EDNS_FRAG));
2516         sq->last_sent_time = *sq->outnet->now_tv;
2517         sq->edns_lame_known = (int)edns_lame_known;
2518         verbose(VERB_ALGO, "serviced query UDP timeout=%d msec", rtt);
2519         sq->pending = pending_udp_query(sq, buff, rtt,
2520                 serviced_udp_callback, sq);
2521         if(!sq->pending)
2522                 return 0;
2523         return 1;
2524 }
2525
2526 /** check that perturbed qname is identical */
2527 static int
2528 serviced_check_qname(sldns_buffer* pkt, uint8_t* qbuf, size_t qbuflen)
2529 {
2530         uint8_t* d1 = sldns_buffer_begin(pkt)+12;
2531         uint8_t* d2 = qbuf+10;
2532         uint8_t len1, len2;
2533         int count = 0;
2534         if(sldns_buffer_limit(pkt) < 12+1+4) /* packet too small for qname */
2535                 return 0;
2536         log_assert(qbuflen >= 15 /* 10 header, root, type, class */);
2537         len1 = *d1++;
2538         len2 = *d2++;
2539         while(len1 != 0 || len2 != 0) {
2540                 if(LABEL_IS_PTR(len1)) {
2541                         /* check if we can read *d1 with compression ptr rest */
2542                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
2543                                 return 0;
2544                         d1 = sldns_buffer_begin(pkt)+PTR_OFFSET(len1, *d1);
2545                         /* check if we can read the destination *d1 */
2546                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
2547                                 return 0;
2548                         len1 = *d1++;
2549                         if(count++ > MAX_COMPRESS_PTRS)
2550                                 return 0;
2551                         continue;
2552                 }
2553                 if(d2 > qbuf+qbuflen)
2554                         return 0;
2555                 if(len1 != len2)
2556                         return 0;
2557                 if(len1 > LDNS_MAX_LABELLEN)
2558                         return 0;
2559                 /* check len1 + 1(next length) are okay to read */
2560                 if(d1+len1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
2561                         return 0;
2562                 log_assert(len1 <= LDNS_MAX_LABELLEN);
2563                 log_assert(len2 <= LDNS_MAX_LABELLEN);
2564                 log_assert(len1 == len2 && len1 != 0);
2565                 /* compare the labels - bitwise identical */
2566                 if(memcmp(d1, d2, len1) != 0)
2567                         return 0;
2568                 d1 += len1;
2569                 d2 += len2;
2570                 len1 = *d1++;
2571                 len2 = *d2++;
2572         }
2573         return 1;
2574 }
2575
2576 /** call the callbacks for a serviced query */
2577 static void
2578 serviced_callbacks(struct serviced_query* sq, int error, struct comm_point* c,
2579         struct comm_reply* rep)
2580 {
2581         struct service_callback* p;
2582         int dobackup = (sq->cblist && sq->cblist->next); /* >1 cb*/
2583         uint8_t *backup_p = NULL;
2584         size_t backlen = 0;
2585 #ifdef UNBOUND_DEBUG
2586         rbnode_type* rem =
2587 #else
2588         (void)
2589 #endif
2590         /* remove from tree, and schedule for deletion, so that callbacks
2591          * can safely deregister themselves and even create new serviced
2592          * queries that are identical to this one. */
2593         rbtree_delete(sq->outnet->serviced, sq);
2594         log_assert(rem); /* should have been present */
2595         sq->to_be_deleted = 1;
2596         verbose(VERB_ALGO, "svcd callbacks start");
2597         if(sq->outnet->use_caps_for_id && error == NETEVENT_NOERROR && c &&
2598                 !sq->nocaps && sq->qtype != LDNS_RR_TYPE_PTR) {
2599                 /* for type PTR do not check perturbed name in answer,
2600                  * compatibility with cisco dns guard boxes that mess up
2601                  * reverse queries 0x20 contents */
2602                 /* noerror and nxdomain must have a qname in reply */
2603                 if(sldns_buffer_read_u16_at(c->buffer, 4) == 0 &&
2604                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
2605                                 == LDNS_RCODE_NOERROR ||
2606                          LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
2607                                 == LDNS_RCODE_NXDOMAIN)) {
2608                         verbose(VERB_DETAIL, "no qname in reply to check 0x20ID");
2609                         log_addr(VERB_DETAIL, "from server",
2610                                 &sq->addr, sq->addrlen);
2611                         log_buf(VERB_DETAIL, "for packet", c->buffer);
2612                         error = NETEVENT_CLOSED;
2613                         c = NULL;
2614                 } else if(sldns_buffer_read_u16_at(c->buffer, 4) > 0 &&
2615                         !serviced_check_qname(c->buffer, sq->qbuf,
2616                         sq->qbuflen)) {
2617                         verbose(VERB_DETAIL, "wrong 0x20-ID in reply qname");
2618                         log_addr(VERB_DETAIL, "from server",
2619                                 &sq->addr, sq->addrlen);
2620                         log_buf(VERB_DETAIL, "for packet", c->buffer);
2621                         error = NETEVENT_CAPSFAIL;
2622                         /* and cleanup too */
2623                         pkt_dname_tolower(c->buffer,
2624                                 sldns_buffer_at(c->buffer, 12));
2625                 } else {
2626                         verbose(VERB_ALGO, "good 0x20-ID in reply qname");
2627                         /* cleanup caps, prettier cache contents. */
2628                         pkt_dname_tolower(c->buffer,
2629                                 sldns_buffer_at(c->buffer, 12));
2630                 }
2631         }
2632         if(dobackup && c) {
2633                 /* make a backup of the query, since the querystate processing
2634                  * may send outgoing queries that overwrite the buffer.
2635                  * use secondary buffer to store the query.
2636                  * This is a data copy, but faster than packet to server */
2637                 backlen = sldns_buffer_limit(c->buffer);
2638                 backup_p = memdup(sldns_buffer_begin(c->buffer), backlen);
2639                 if(!backup_p) {
2640                         log_err("malloc failure in serviced query callbacks");
2641                         error = NETEVENT_CLOSED;
2642                         c = NULL;
2643                 }
2644                 sq->outnet->svcd_overhead = backlen;
2645         }
2646         /* test the actual sq->cblist, because the next elem could be deleted*/
2647         while((p=sq->cblist) != NULL) {
2648                 sq->cblist = p->next; /* remove this element */
2649                 if(dobackup && c) {
2650                         sldns_buffer_clear(c->buffer);
2651                         sldns_buffer_write(c->buffer, backup_p, backlen);
2652                         sldns_buffer_flip(c->buffer);
2653                 }
2654                 fptr_ok(fptr_whitelist_serviced_query(p->cb));
2655                 (void)(*p->cb)(c, p->cb_arg, error, rep);
2656                 free(p);
2657         }
2658         if(backup_p) {
2659                 free(backup_p);
2660                 sq->outnet->svcd_overhead = 0;
2661         }
2662         verbose(VERB_ALGO, "svcd callbacks end");
2663         log_assert(sq->cblist == NULL);
2664         serviced_delete(sq);
2665 }
2666
2667 int
2668 serviced_tcp_callback(struct comm_point* c, void* arg, int error,
2669         struct comm_reply* rep)
2670 {
2671         struct serviced_query* sq = (struct serviced_query*)arg;
2672         struct comm_reply r2;
2673         sq->pending = NULL; /* removed after this callback */
2674         if(error != NETEVENT_NOERROR)
2675                 log_addr(VERB_QUERY, "tcp error for address",
2676                         &sq->addr, sq->addrlen);
2677         if(error==NETEVENT_NOERROR)
2678                 infra_update_tcp_works(sq->outnet->infra, &sq->addr,
2679                         sq->addrlen, sq->zone, sq->zonelen);
2680 #ifdef USE_DNSTAP
2681         if(error==NETEVENT_NOERROR && sq->outnet->dtenv &&
2682            (sq->outnet->dtenv->log_resolver_response_messages ||
2683             sq->outnet->dtenv->log_forwarder_response_messages))
2684                 dt_msg_send_outside_response(sq->outnet->dtenv, &sq->addr,
2685                 c->type, sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
2686                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
2687 #endif
2688         if(error==NETEVENT_NOERROR && sq->status == serviced_query_TCP_EDNS &&
2689                 (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
2690                 LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(sldns_buffer_begin(
2691                 c->buffer)) == LDNS_RCODE_NOTIMPL) ) {
2692                 /* attempt to fallback to nonEDNS */
2693                 sq->status = serviced_query_TCP_EDNS_fallback;
2694                 serviced_tcp_initiate(sq, c->buffer);
2695                 return 0;
2696         } else if(error==NETEVENT_NOERROR &&
2697                 sq->status == serviced_query_TCP_EDNS_fallback &&
2698                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
2699                         LDNS_RCODE_NOERROR || LDNS_RCODE_WIRE(
2700                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NXDOMAIN
2701                         || LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
2702                         == LDNS_RCODE_YXDOMAIN)) {
2703                 /* the fallback produced a result that looks promising, note
2704                  * that this server should be approached without EDNS */
2705                 /* only store noEDNS in cache if domain is noDNSSEC */
2706                 if(!sq->want_dnssec)
2707                   if(!infra_edns_update(sq->outnet->infra, &sq->addr,
2708                         sq->addrlen, sq->zone, sq->zonelen, -1,
2709                         *sq->outnet->now_secs))
2710                         log_err("Out of memory caching no edns for host");
2711                 sq->status = serviced_query_TCP;
2712         }
2713         if(sq->tcp_upstream || sq->ssl_upstream) {
2714             struct timeval now = *sq->outnet->now_tv;
2715             if(error!=NETEVENT_NOERROR) {
2716                 if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
2717                     sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
2718                     -1, sq->last_rtt, (time_t)now.tv_sec))
2719                     log_err("out of memory in TCP exponential backoff.");
2720             } else if(now.tv_sec > sq->last_sent_time.tv_sec ||
2721                 (now.tv_sec == sq->last_sent_time.tv_sec &&
2722                 now.tv_usec > sq->last_sent_time.tv_usec)) {
2723                 /* convert from microseconds to milliseconds */
2724                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
2725                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
2726                 verbose(VERB_ALGO, "measured TCP-time at %d msec", roundtime);
2727                 log_assert(roundtime >= 0);
2728                 /* only store if less then AUTH_TIMEOUT seconds, it could be
2729                  * huge due to system-hibernated and we woke up */
2730                 if(roundtime < 60000) {
2731                     if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
2732                         sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
2733                         roundtime, sq->last_rtt, (time_t)now.tv_sec))
2734                         log_err("out of memory noting rtt.");
2735                 }
2736             }
2737         }
2738         /* insert address into reply info */
2739         if(!rep) {
2740                 /* create one if there isn't (on errors) */
2741                 rep = &r2;
2742                 r2.c = c;
2743         }
2744         memcpy(&rep->addr, &sq->addr, sq->addrlen);
2745         rep->addrlen = sq->addrlen;
2746         serviced_callbacks(sq, error, c, rep);
2747         return 0;
2748 }
2749
2750 static void
2751 serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
2752 {
2753         verbose(VERB_ALGO, "initiate TCP query %s",
2754                 sq->status==serviced_query_TCP_EDNS?"EDNS":"");
2755         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
2756         sq->last_sent_time = *sq->outnet->now_tv;
2757         sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
2758                 serviced_tcp_callback, sq);
2759         if(!sq->pending) {
2760                 /* delete from tree so that a retry by above layer does not
2761                  * clash with this entry */
2762                 verbose(VERB_ALGO, "serviced_tcp_initiate: failed to send tcp query");
2763                 serviced_callbacks(sq, NETEVENT_CLOSED, NULL, NULL);
2764         }
2765 }
2766
2767 /** Send serviced query over TCP return false on initial failure */
2768 static int
2769 serviced_tcp_send(struct serviced_query* sq, sldns_buffer* buff)
2770 {
2771         int vs, rtt, timeout;
2772         uint8_t edns_lame_known;
2773         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
2774                 sq->zonelen, *sq->outnet->now_secs, &vs, &edns_lame_known,
2775                 &rtt))
2776                 return 0;
2777         sq->last_rtt = rtt;
2778         if(vs != -1)
2779                 sq->status = serviced_query_TCP_EDNS;
2780         else    sq->status = serviced_query_TCP;
2781         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
2782         sq->last_sent_time = *sq->outnet->now_tv;
2783         if(sq->tcp_upstream || sq->ssl_upstream) {
2784                 timeout = rtt;
2785                 if(rtt >= UNKNOWN_SERVER_NICENESS && rtt < TCP_AUTH_QUERY_TIMEOUT)
2786                         timeout = TCP_AUTH_QUERY_TIMEOUT;
2787         } else {
2788                 timeout = TCP_AUTH_QUERY_TIMEOUT;
2789         }
2790         sq->pending = pending_tcp_query(sq, buff, timeout,
2791                 serviced_tcp_callback, sq);
2792         return sq->pending != NULL;
2793 }
2794
2795 /* see if packet is edns malformed; got zeroes at start.
2796  * This is from servers that return malformed packets to EDNS0 queries,
2797  * but they return good packets for nonEDNS0 queries.
2798  * We try to detect their output; without resorting to a full parse or
2799  * check for too many bytes after the end of the packet. */
2800 static int
2801 packet_edns_malformed(struct sldns_buffer* buf, int qtype)
2802 {
2803         size_t len;
2804         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE)
2805                 return 1; /* malformed */
2806         /* they have NOERROR rcode, 1 answer. */
2807         if(LDNS_RCODE_WIRE(sldns_buffer_begin(buf)) != LDNS_RCODE_NOERROR)
2808                 return 0;
2809         /* one query (to skip) and answer records */
2810         if(LDNS_QDCOUNT(sldns_buffer_begin(buf)) != 1 ||
2811                 LDNS_ANCOUNT(sldns_buffer_begin(buf)) == 0)
2812                 return 0;
2813         /* skip qname */
2814         len = dname_valid(sldns_buffer_at(buf, LDNS_HEADER_SIZE),
2815                 sldns_buffer_limit(buf)-LDNS_HEADER_SIZE);
2816         if(len == 0)
2817                 return 0;
2818         if(len == 1 && qtype == 0)
2819                 return 0; /* we asked for '.' and type 0 */
2820         /* and then 4 bytes (type and class of query) */
2821         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE + len + 4 + 3)
2822                 return 0;
2823
2824         /* and start with 11 zeroes as the answer RR */
2825         /* so check the qtype of the answer record, qname=0, type=0 */
2826         if(sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[0] == 0 &&
2827            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[1] == 0 &&
2828            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[2] == 0)
2829                 return 1;
2830         return 0;
2831 }
2832
2833 int
2834 serviced_udp_callback(struct comm_point* c, void* arg, int error,
2835         struct comm_reply* rep)
2836 {
2837         struct serviced_query* sq = (struct serviced_query*)arg;
2838         struct outside_network* outnet = sq->outnet;
2839         struct timeval now = *sq->outnet->now_tv;
2840
2841         sq->pending = NULL; /* removed after callback */
2842         if(error == NETEVENT_TIMEOUT) {
2843                 if(sq->status == serviced_query_UDP_EDNS && sq->last_rtt < 5000) {
2844                         /* fallback to 1480/1280 */
2845                         sq->status = serviced_query_UDP_EDNS_FRAG;
2846                         log_name_addr(VERB_ALGO, "try edns1xx0", sq->qbuf+10,
2847                                 &sq->addr, sq->addrlen);
2848                         if(!serviced_udp_send(sq, c->buffer)) {
2849                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
2850                         }
2851                         return 0;
2852                 }
2853                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
2854                         /* fragmentation size did not fix it */
2855                         sq->status = serviced_query_UDP_EDNS;
2856                 }
2857                 sq->retry++;
2858                 if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
2859                         sq->zone, sq->zonelen, sq->qtype, -1, sq->last_rtt,
2860                         (time_t)now.tv_sec))
2861                         log_err("out of memory in UDP exponential backoff");
2862                 if(sq->retry < OUTBOUND_UDP_RETRY) {
2863                         log_name_addr(VERB_ALGO, "retry query", sq->qbuf+10,
2864                                 &sq->addr, sq->addrlen);
2865                         if(!serviced_udp_send(sq, c->buffer)) {
2866                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
2867                         }
2868                         return 0;
2869                 }
2870         }
2871         if(error != NETEVENT_NOERROR) {
2872                 /* udp returns error (due to no ID or interface available) */
2873                 serviced_callbacks(sq, error, c, rep);
2874                 return 0;
2875         }
2876 #ifdef USE_DNSTAP
2877         if(error == NETEVENT_NOERROR && outnet->dtenv &&
2878            (outnet->dtenv->log_resolver_response_messages ||
2879             outnet->dtenv->log_forwarder_response_messages))
2880                 dt_msg_send_outside_response(outnet->dtenv, &sq->addr, c->type,
2881                 sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
2882                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
2883 #endif
2884         if( (sq->status == serviced_query_UDP_EDNS
2885                 ||sq->status == serviced_query_UDP_EDNS_FRAG)
2886                 && (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
2887                         == LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(
2888                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOTIMPL
2889                     || packet_edns_malformed(c->buffer, sq->qtype)
2890                         )) {
2891                 /* try to get an answer by falling back without EDNS */
2892                 verbose(VERB_ALGO, "serviced query: attempt without EDNS");
2893                 sq->status = serviced_query_UDP_EDNS_fallback;
2894                 sq->retry = 0;
2895                 if(!serviced_udp_send(sq, c->buffer)) {
2896                         serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
2897                 }
2898                 return 0;
2899         } else if(sq->status == serviced_query_UDP_EDNS &&
2900                 !sq->edns_lame_known) {
2901                 /* now we know that edns queries received answers store that */
2902                 log_addr(VERB_ALGO, "serviced query: EDNS works for",
2903                         &sq->addr, sq->addrlen);
2904                 if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
2905                         sq->zone, sq->zonelen, 0, (time_t)now.tv_sec)) {
2906                         log_err("Out of memory caching edns works");
2907                 }
2908                 sq->edns_lame_known = 1;
2909         } else if(sq->status == serviced_query_UDP_EDNS_fallback &&
2910                 !sq->edns_lame_known && (LDNS_RCODE_WIRE(
2911                 sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOERROR ||
2912                 LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
2913                 LDNS_RCODE_NXDOMAIN || LDNS_RCODE_WIRE(sldns_buffer_begin(
2914                 c->buffer)) == LDNS_RCODE_YXDOMAIN)) {
2915                 /* the fallback produced a result that looks promising, note
2916                  * that this server should be approached without EDNS */
2917                 /* only store noEDNS in cache if domain is noDNSSEC */
2918                 if(!sq->want_dnssec) {
2919                   log_addr(VERB_ALGO, "serviced query: EDNS fails for",
2920                         &sq->addr, sq->addrlen);
2921                   if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
2922                         sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
2923                         log_err("Out of memory caching no edns for host");
2924                   }
2925                 } else {
2926                   log_addr(VERB_ALGO, "serviced query: EDNS fails, but "
2927                         "not stored because need DNSSEC for", &sq->addr,
2928                         sq->addrlen);
2929                 }
2930                 sq->status = serviced_query_UDP;
2931         }
2932         if(now.tv_sec > sq->last_sent_time.tv_sec ||
2933                 (now.tv_sec == sq->last_sent_time.tv_sec &&
2934                 now.tv_usec > sq->last_sent_time.tv_usec)) {
2935                 /* convert from microseconds to milliseconds */
2936                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
2937                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
2938                 verbose(VERB_ALGO, "measured roundtrip at %d msec", roundtime);
2939                 log_assert(roundtime >= 0);
2940                 /* in case the system hibernated, do not enter a huge value,
2941                  * above this value gives trouble with server selection */
2942                 if(roundtime < 60000) {
2943                     if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
2944                         sq->zone, sq->zonelen, sq->qtype, roundtime,
2945                         sq->last_rtt, (time_t)now.tv_sec))
2946                         log_err("out of memory noting rtt.");
2947                 }
2948         }
2949         /* perform TC flag check and TCP fallback after updating our
2950          * cache entries for EDNS status and RTT times */
2951         if(LDNS_TC_WIRE(sldns_buffer_begin(c->buffer))) {
2952                 /* fallback to TCP */
2953                 /* this discards partial UDP contents */
2954                 if(sq->status == serviced_query_UDP_EDNS ||
2955                         sq->status == serviced_query_UDP_EDNS_FRAG ||
2956                         sq->status == serviced_query_UDP_EDNS_fallback)
2957                         /* if we have unfinished EDNS_fallback, start again */
2958                         sq->status = serviced_query_TCP_EDNS;
2959                 else    sq->status = serviced_query_TCP;
2960                 serviced_tcp_initiate(sq, c->buffer);
2961                 return 0;
2962         }
2963         /* yay! an answer */
2964         serviced_callbacks(sq, error, c, rep);
2965         return 0;
2966 }
2967
2968 struct serviced_query*
2969 outnet_serviced_query(struct outside_network* outnet,
2970         struct query_info* qinfo, uint16_t flags, int dnssec, int want_dnssec,
2971         int nocaps, int tcp_upstream, int ssl_upstream, char* tls_auth_name,
2972         struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
2973         size_t zonelen, struct module_qstate* qstate,
2974         comm_point_callback_type* callback, void* callback_arg, sldns_buffer* buff,
2975         struct module_env* env)
2976 {
2977         struct serviced_query* sq;
2978         struct service_callback* cb;
2979         struct edns_string_addr* client_string_addr;
2980
2981         if(!inplace_cb_query_call(env, qinfo, flags, addr, addrlen, zone, zonelen,
2982                 qstate, qstate->region))
2983                         return NULL;
2984
2985         if((client_string_addr = edns_string_addr_lookup(
2986                 &env->edns_strings->client_strings, addr, addrlen))) {
2987                 edns_opt_list_append(&qstate->edns_opts_back_out,
2988                         env->edns_strings->client_string_opcode,
2989                         client_string_addr->string_len,
2990                         client_string_addr->string, qstate->region);
2991         }
2992
2993         serviced_gen_query(buff, qinfo->qname, qinfo->qname_len, qinfo->qtype,
2994                 qinfo->qclass, flags);
2995         sq = lookup_serviced(outnet, buff, dnssec, addr, addrlen,
2996                 qstate->edns_opts_back_out);
2997         /* duplicate entries are included in the callback list, because
2998          * there is a counterpart registration by our caller that needs to
2999          * be doubly-removed (with callbacks perhaps). */
3000         if(!(cb = (struct service_callback*)malloc(sizeof(*cb))))
3001                 return NULL;
3002         if(!sq) {
3003                 /* make new serviced query entry */
3004                 sq = serviced_create(outnet, buff, dnssec, want_dnssec, nocaps,
3005                         tcp_upstream, ssl_upstream, tls_auth_name, addr,
3006                         addrlen, zone, zonelen, (int)qinfo->qtype,
3007                         qstate->edns_opts_back_out);
3008                 if(!sq) {
3009                         free(cb);
3010                         return NULL;
3011                 }
3012                 /* perform first network action */
3013                 if(outnet->do_udp && !(tcp_upstream || ssl_upstream)) {
3014                         if(!serviced_udp_send(sq, buff)) {
3015                                 (void)rbtree_delete(outnet->serviced, sq);
3016                                 serviced_node_del(&sq->node, NULL);
3017                                 free(cb);
3018                                 return NULL;
3019                         }
3020                 } else {
3021                         if(!serviced_tcp_send(sq, buff)) {
3022                                 (void)rbtree_delete(outnet->serviced, sq);
3023                                 serviced_node_del(&sq->node, NULL);
3024                                 free(cb);
3025                                 return NULL;
3026                         }
3027                 }
3028         }
3029         /* add callback to list of callbacks */
3030         cb->cb = callback;
3031         cb->cb_arg = callback_arg;
3032         cb->next = sq->cblist;
3033         sq->cblist = cb;
3034         return sq;
3035 }
3036
3037 /** remove callback from list */
3038 static void
3039 callback_list_remove(struct serviced_query* sq, void* cb_arg)
3040 {
3041         struct service_callback** pp = &sq->cblist;
3042         while(*pp) {
3043                 if((*pp)->cb_arg == cb_arg) {
3044                         struct service_callback* del = *pp;
3045                         *pp = del->next;
3046                         free(del);
3047                         return;
3048                 }
3049                 pp = &(*pp)->next;
3050         }
3051 }
3052
3053 void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
3054 {
3055         if(!sq)
3056                 return;
3057         callback_list_remove(sq, cb_arg);
3058         /* if callbacks() routine scheduled deletion, let it do that */
3059         if(!sq->cblist && !sq->to_be_deleted) {
3060                 (void)rbtree_delete(sq->outnet->serviced, sq);
3061                 serviced_delete(sq);
3062         }
3063 }
3064
3065 /** create fd to send to this destination */
3066 static int
3067 fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr,
3068         socklen_t to_addrlen)
3069 {
3070         struct sockaddr_storage* addr;
3071         socklen_t addrlen;
3072         int i, try, pnum, dscp;
3073         struct port_if* pif;
3074
3075         /* create fd */
3076         dscp = outnet->ip_dscp;
3077         for(try = 0; try<1000; try++) {
3078                 int port = 0;
3079                 int freebind = 0;
3080                 int noproto = 0;
3081                 int inuse = 0;
3082                 int fd = -1;
3083
3084                 /* select interface */
3085                 if(addr_is_ip6(to_addr, to_addrlen)) {
3086                         if(outnet->num_ip6 == 0) {
3087                                 char to[64];
3088                                 addr_to_str(to_addr, to_addrlen, to, sizeof(to));
3089                                 verbose(VERB_QUERY, "need ipv6 to send, but no ipv6 outgoing interfaces, for %s", to);
3090                                 return -1;
3091                         }
3092                         i = ub_random_max(outnet->rnd, outnet->num_ip6);
3093                         pif = &outnet->ip6_ifs[i];
3094                 } else {
3095                         if(outnet->num_ip4 == 0) {
3096                                 char to[64];
3097                                 addr_to_str(to_addr, to_addrlen, to, sizeof(to));
3098                                 verbose(VERB_QUERY, "need ipv4 to send, but no ipv4 outgoing interfaces, for %s", to);
3099                                 return -1;
3100                         }
3101                         i = ub_random_max(outnet->rnd, outnet->num_ip4);
3102                         pif = &outnet->ip4_ifs[i];
3103                 }
3104                 addr = &pif->addr;
3105                 addrlen = pif->addrlen;
3106 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
3107                 pnum = ub_random_max(outnet->rnd, pif->avail_total);
3108                 if(pnum < pif->inuse) {
3109                         /* port already open */
3110                         port = pif->out[pnum]->number;
3111                 } else {
3112                         /* unused ports in start part of array */
3113                         port = pif->avail_ports[pnum - pif->inuse];
3114                 }
3115 #else
3116                 pnum = port = 0;
3117 #endif
3118                 if(addr_is_ip6(to_addr, to_addrlen)) {
3119                         struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
3120                         sa.sin6_port = (in_port_t)htons((uint16_t)port);
3121                         fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
3122                                 (struct sockaddr*)&sa, addrlen, 1, &inuse, &noproto,
3123                                 0, 0, 0, NULL, 0, freebind, 0, dscp);
3124                 } else {
3125                         struct sockaddr_in* sa = (struct sockaddr_in*)addr;
3126                         sa->sin_port = (in_port_t)htons((uint16_t)port);
3127                         fd = create_udp_sock(AF_INET, SOCK_DGRAM,
3128                                 (struct sockaddr*)addr, addrlen, 1, &inuse, &noproto,
3129                                 0, 0, 0, NULL, 0, freebind, 0, dscp);
3130                 }
3131                 if(fd != -1) {
3132                         return fd;
3133                 }
3134                 if(!inuse) {
3135                         return -1;
3136                 }
3137         }
3138         /* too many tries */
3139         log_err("cannot send probe, ports are in use");
3140         return -1;
3141 }
3142
3143 struct comm_point*
3144 outnet_comm_point_for_udp(struct outside_network* outnet,
3145         comm_point_callback_type* cb, void* cb_arg,
3146         struct sockaddr_storage* to_addr, socklen_t to_addrlen)
3147 {
3148         struct comm_point* cp;
3149         int fd = fd_for_dest(outnet, to_addr, to_addrlen);
3150         if(fd == -1) {
3151                 return NULL;
3152         }
3153         cp = comm_point_create_udp(outnet->base, fd, outnet->udp_buff,
3154                 cb, cb_arg);
3155         if(!cp) {
3156                 log_err("malloc failure");
3157                 close(fd);
3158                 return NULL;
3159         }
3160         return cp;
3161 }
3162
3163 /** setup SSL for comm point */
3164 static int
3165 setup_comm_ssl(struct comm_point* cp, struct outside_network* outnet,
3166         int fd, char* host)
3167 {
3168         cp->ssl = outgoing_ssl_fd(outnet->sslctx, fd);
3169         if(!cp->ssl) {
3170                 log_err("cannot create SSL object");
3171                 return 0;
3172         }
3173 #ifdef USE_WINSOCK
3174         comm_point_tcp_win_bio_cb(cp, cp->ssl);
3175 #endif
3176         cp->ssl_shake_state = comm_ssl_shake_write;
3177         /* https verification */
3178 #ifdef HAVE_SSL
3179         if(outnet->tls_use_sni) {
3180                 (void)SSL_set_tlsext_host_name(cp->ssl, host);
3181         }
3182 #endif
3183 #ifdef HAVE_SSL_SET1_HOST
3184         if((SSL_CTX_get_verify_mode(outnet->sslctx)&SSL_VERIFY_PEER)) {
3185                 /* because we set SSL_VERIFY_PEER, in netevent in
3186                  * ssl_handshake, it'll check if the certificate
3187                  * verification has succeeded */
3188                 /* SSL_VERIFY_PEER is set on the sslctx */
3189                 /* and the certificates to verify with are loaded into
3190                  * it with SSL_load_verify_locations or
3191                  * SSL_CTX_set_default_verify_paths */
3192                 /* setting the hostname makes openssl verify the
3193                  * host name in the x509 certificate in the
3194                  * SSL connection*/
3195                 if(!SSL_set1_host(cp->ssl, host)) {
3196                         log_err("SSL_set1_host failed");
3197                         return 0;
3198                 }
3199         }
3200 #elif defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
3201         /* openssl 1.0.2 has this function that can be used for
3202          * set1_host like verification */
3203         if((SSL_CTX_get_verify_mode(outnet->sslctx)&SSL_VERIFY_PEER)) {
3204                 X509_VERIFY_PARAM* param = SSL_get0_param(cp->ssl);
3205 #  ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS
3206                 X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
3207 #  endif
3208                 if(!X509_VERIFY_PARAM_set1_host(param, host, strlen(host))) {
3209                         log_err("X509_VERIFY_PARAM_set1_host failed");
3210                         return 0;
3211                 }
3212         }
3213 #else
3214         (void)host;
3215 #endif /* HAVE_SSL_SET1_HOST */
3216         return 1;
3217 }
3218
3219 struct comm_point*
3220 outnet_comm_point_for_tcp(struct outside_network* outnet,
3221         comm_point_callback_type* cb, void* cb_arg,
3222         struct sockaddr_storage* to_addr, socklen_t to_addrlen,
3223         sldns_buffer* query, int timeout, int ssl, char* host)
3224 {
3225         struct comm_point* cp;
3226         int fd = outnet_get_tcp_fd(to_addr, to_addrlen, outnet->tcp_mss, outnet->ip_dscp);
3227         if(fd == -1) {
3228                 return 0;
3229         }
3230         fd_set_nonblock(fd);
3231         if(!outnet_tcp_connect(fd, to_addr, to_addrlen)) {
3232                 /* outnet_tcp_connect has closed fd on error for us */
3233                 return 0;
3234         }
3235         cp = comm_point_create_tcp_out(outnet->base, 65552, cb, cb_arg);
3236         if(!cp) {
3237                 log_err("malloc failure");
3238                 close(fd);
3239                 return 0;
3240         }
3241         cp->repinfo.addrlen = to_addrlen;
3242         memcpy(&cp->repinfo.addr, to_addr, to_addrlen);
3243
3244         /* setup for SSL (if needed) */
3245         if(ssl) {
3246                 if(!setup_comm_ssl(cp, outnet, fd, host)) {
3247                         log_err("cannot setup XoT");
3248                         comm_point_delete(cp);
3249                         return NULL;
3250                 }
3251         }
3252
3253         /* set timeout on TCP connection */
3254         comm_point_start_listening(cp, fd, timeout);
3255         /* copy scratch buffer to cp->buffer */
3256         sldns_buffer_copy(cp->buffer, query);
3257         return cp;
3258 }
3259
3260 /** setup http request headers in buffer for sending query to destination */
3261 static int
3262 setup_http_request(sldns_buffer* buf, char* host, char* path)
3263 {
3264         sldns_buffer_clear(buf);
3265         sldns_buffer_printf(buf, "GET /%s HTTP/1.1\r\n", path);
3266         sldns_buffer_printf(buf, "Host: %s\r\n", host);
3267         sldns_buffer_printf(buf, "User-Agent: unbound/%s\r\n",
3268                 PACKAGE_VERSION);
3269         /* We do not really do multiple queries per connection,
3270          * but this header setting is also not needed.
3271          * sldns_buffer_printf(buf, "Connection: close\r\n") */
3272         sldns_buffer_printf(buf, "\r\n");
3273         if(sldns_buffer_position(buf)+10 > sldns_buffer_capacity(buf))
3274                 return 0; /* somehow buffer too short, but it is about 60K
3275                 and the request is only a couple bytes long. */
3276         sldns_buffer_flip(buf);
3277         return 1;
3278 }
3279
3280 struct comm_point*
3281 outnet_comm_point_for_http(struct outside_network* outnet,
3282         comm_point_callback_type* cb, void* cb_arg,
3283         struct sockaddr_storage* to_addr, socklen_t to_addrlen, int timeout,
3284         int ssl, char* host, char* path)
3285 {
3286         /* cp calls cb with err=NETEVENT_DONE when transfer is done */
3287         struct comm_point* cp;
3288         int fd = outnet_get_tcp_fd(to_addr, to_addrlen, outnet->tcp_mss, outnet->ip_dscp);
3289         if(fd == -1) {
3290                 return 0;
3291         }
3292         fd_set_nonblock(fd);
3293         if(!outnet_tcp_connect(fd, to_addr, to_addrlen)) {
3294                 /* outnet_tcp_connect has closed fd on error for us */
3295                 return 0;
3296         }
3297         cp = comm_point_create_http_out(outnet->base, 65552, cb, cb_arg,
3298                 outnet->udp_buff);
3299         if(!cp) {
3300                 log_err("malloc failure");
3301                 close(fd);
3302                 return 0;
3303         }
3304         cp->repinfo.addrlen = to_addrlen;
3305         memcpy(&cp->repinfo.addr, to_addr, to_addrlen);
3306
3307         /* setup for SSL (if needed) */
3308         if(ssl) {
3309                 if(!setup_comm_ssl(cp, outnet, fd, host)) {
3310                         log_err("cannot setup https");
3311                         comm_point_delete(cp);
3312                         return NULL;
3313                 }
3314         }
3315
3316         /* set timeout on TCP connection */
3317         comm_point_start_listening(cp, fd, timeout);
3318
3319         /* setup http request in cp->buffer */
3320         if(!setup_http_request(cp->buffer, host, path)) {
3321                 log_err("error setting up http request");
3322                 comm_point_delete(cp);
3323                 return NULL;
3324         }
3325         return cp;
3326 }
3327
3328 /** get memory used by waiting tcp entry (in use or not) */
3329 static size_t
3330 waiting_tcp_get_mem(struct waiting_tcp* w)
3331 {
3332         size_t s;
3333         if(!w) return 0;
3334         s = sizeof(*w) + w->pkt_len;
3335         if(w->timer)
3336                 s += comm_timer_get_mem(w->timer);
3337         return s;
3338 }
3339
3340 /** get memory used by port if */
3341 static size_t
3342 if_get_mem(struct port_if* pif)
3343 {
3344         size_t s;
3345         int i;
3346         s = sizeof(*pif) +
3347 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
3348             sizeof(int)*pif->avail_total +
3349 #endif
3350                 sizeof(struct port_comm*)*pif->maxout;
3351         for(i=0; i<pif->inuse; i++)
3352                 s += sizeof(*pif->out[i]) +
3353                         comm_point_get_mem(pif->out[i]->cp);
3354         return s;
3355 }
3356
3357 /** get memory used by waiting udp */
3358 static size_t
3359 waiting_udp_get_mem(struct pending* w)
3360 {
3361         size_t s;
3362         s = sizeof(*w) + comm_timer_get_mem(w->timer) + w->pkt_len;
3363         return s;
3364 }
3365
3366 size_t outnet_get_mem(struct outside_network* outnet)
3367 {
3368         size_t i;
3369         int k;
3370         struct waiting_tcp* w;
3371         struct pending* u;
3372         struct serviced_query* sq;
3373         struct service_callback* sb;
3374         struct port_comm* pc;
3375         size_t s = sizeof(*outnet) + sizeof(*outnet->base) +
3376                 sizeof(*outnet->udp_buff) +
3377                 sldns_buffer_capacity(outnet->udp_buff);
3378         /* second buffer is not ours */
3379         for(pc = outnet->unused_fds; pc; pc = pc->next) {
3380                 s += sizeof(*pc) + comm_point_get_mem(pc->cp);
3381         }
3382         for(k=0; k<outnet->num_ip4; k++)
3383                 s += if_get_mem(&outnet->ip4_ifs[k]);
3384         for(k=0; k<outnet->num_ip6; k++)
3385                 s += if_get_mem(&outnet->ip6_ifs[k]);
3386         for(u=outnet->udp_wait_first; u; u=u->next_waiting)
3387                 s += waiting_udp_get_mem(u);
3388
3389         s += sizeof(struct pending_tcp*)*outnet->num_tcp;
3390         for(i=0; i<outnet->num_tcp; i++) {
3391                 s += sizeof(struct pending_tcp);
3392                 s += comm_point_get_mem(outnet->tcp_conns[i]->c);
3393                 if(outnet->tcp_conns[i]->query)
3394                         s += waiting_tcp_get_mem(outnet->tcp_conns[i]->query);
3395         }
3396         for(w=outnet->tcp_wait_first; w; w = w->next_waiting)
3397                 s += waiting_tcp_get_mem(w);
3398         s += sizeof(*outnet->pending);
3399         s += (sizeof(struct pending) + comm_timer_get_mem(NULL)) *
3400                 outnet->pending->count;
3401         s += sizeof(*outnet->serviced);
3402         s += outnet->svcd_overhead;
3403         RBTREE_FOR(sq, struct serviced_query*, outnet->serviced) {
3404                 s += sizeof(*sq) + sq->qbuflen;
3405                 for(sb = sq->cblist; sb; sb = sb->next)
3406                         s += sizeof(*sb);
3407         }
3408         return s;
3409 }
3410
3411 size_t
3412 serviced_get_mem(struct serviced_query* sq)
3413 {
3414         struct service_callback* sb;
3415         size_t s;
3416         s = sizeof(*sq) + sq->qbuflen;
3417         for(sb = sq->cblist; sb; sb = sb->next)
3418                 s += sizeof(*sb);
3419         if(sq->status == serviced_query_UDP_EDNS ||
3420                 sq->status == serviced_query_UDP ||
3421                 sq->status == serviced_query_UDP_EDNS_FRAG ||
3422                 sq->status == serviced_query_UDP_EDNS_fallback) {
3423                 s += sizeof(struct pending);
3424                 s += comm_timer_get_mem(NULL);
3425         } else {
3426                 /* does not have size of the pkt pointer */
3427                 /* always has a timer except on malloc failures */
3428
3429                 /* these sizes are part of the main outside network mem */
3430                 /*
3431                 s += sizeof(struct waiting_tcp);
3432                 s += comm_timer_get_mem(NULL);
3433                 */
3434         }
3435         return s;
3436 }
3437