contrib/unbound/services/outside_network.c

   1 /*
   2  * services/outside_network.c - implement sending of queries and wait answer.
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file has functions to send queries to authoritative servers and
  40  * wait for the pending answer events.
  41  */
  42 #include "config.h"
  43 #include <ctype.h>
  44 #ifdef HAVE_SYS_TYPES_H
  45 #  include <sys/types.h>
  46 #endif
  47 #include <sys/time.h>
  48 #include "services/outside_network.h"
  49 #include "services/listen_dnsport.h"
  50 #include "services/cache/infra.h"
  51 #include "iterator/iterator.h"
  52 #include "util/data/msgparse.h"
  53 #include "util/data/msgreply.h"
  54 #include "util/data/msgencode.h"
  55 #include "util/data/dname.h"
  56 #include "util/netevent.h"
  57 #include "util/log.h"
  58 #include "util/net_help.h"
  59 #include "util/random.h"
  60 #include "util/fptr_wlist.h"
  61 #include "util/edns.h"
  62 #include "sldns/sbuffer.h"
  63 #include "dnstap/dnstap.h"
  64 #ifdef HAVE_OPENSSL_SSL_H
  65 #include <openssl/ssl.h>
  66 #endif
  67 #ifdef HAVE_X509_VERIFY_PARAM_SET1_HOST
  68 #include <openssl/x509v3.h>
  69 #endif
  70
  71 #ifdef HAVE_NETDB_H
  72 #include <netdb.h>
  73 #endif
  74 #include <fcntl.h>
  75
  76 /** number of times to retry making a random ID that is unique. */
  77 #define MAX_ID_RETRY 1000
  78 /** number of times to retry finding interface, port that can be opened. */
  79 #define MAX_PORT_RETRY 10000
  80 /** number of retries on outgoing UDP queries */
  81 #define OUTBOUND_UDP_RETRY 1
  82
  83 /** initiate TCP transaction for serviced query */
  84 static void serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff);
  85 /** with a fd available, randomize and send UDP */
  86 static int randomize_and_send_udp(struct pending* pend, sldns_buffer* packet,
  87         int timeout);
  88
  89 /** remove waiting tcp from the outnet waiting list */
  90 static void waiting_list_remove(struct outside_network* outnet,
  91         struct waiting_tcp* w);
  92
  93 int
  94 pending_cmp(const void* key1, const void* key2)
  95 {
  96         struct pending *p1 = (struct pending*)key1;
  97         struct pending *p2 = (struct pending*)key2;
  98         if(p1->id < p2->id)
  99                 return -1;
 100         if(p1->id > p2->id)
 101                 return 1;
 102         log_assert(p1->id == p2->id);
 103         return sockaddr_cmp(&p1->addr, p1->addrlen, &p2->addr, p2->addrlen);
 104 }
 105
 106 int
 107 serviced_cmp(const void* key1, const void* key2)
 108 {
 109         struct serviced_query* q1 = (struct serviced_query*)key1;
 110         struct serviced_query* q2 = (struct serviced_query*)key2;
 111         int r;
 112         if(q1->qbuflen < q2->qbuflen)
 113                 return -1;
 114         if(q1->qbuflen > q2->qbuflen)
 115                 return 1;
 116         log_assert(q1->qbuflen == q2->qbuflen);
 117         log_assert(q1->qbuflen >= 15 /* 10 header, root, type, class */);
 118         /* alternate casing of qname is still the same query */
 119         if((r = memcmp(q1->qbuf, q2->qbuf, 10)) != 0)
 120                 return r;
 121         if((r = memcmp(q1->qbuf+q1->qbuflen-4, q2->qbuf+q2->qbuflen-4, 4)) != 0)
 122                 return r;
 123         if(q1->dnssec != q2->dnssec) {
 124                 if(q1->dnssec < q2->dnssec)
 125                         return -1;
 126                 return 1;
 127         }
 128         if((r = query_dname_compare(q1->qbuf+10, q2->qbuf+10)) != 0)
 129                 return r;
 130         if((r = edns_opt_list_compare(q1->opt_list, q2->opt_list)) != 0)
 131                 return r;
 132         return sockaddr_cmp(&q1->addr, q1->addrlen, &q2->addr, q2->addrlen);
 133 }
 134
 135 /** delete waiting_tcp entry. Does not unlink from waiting list.
 136  * @param w: to delete.
 137  */
 138 static void
 139 waiting_tcp_delete(struct waiting_tcp* w)
 140 {
 141         if(!w) return;
 142         if(w->timer)
 143                 comm_timer_delete(w->timer);
 144         free(w);
 145 }
 146
 147 /**
 148  * Pick random outgoing-interface of that family, and bind it.
 149  * port set to 0 so OS picks a port number for us.
 150  * if it is the ANY address, do not bind.
 151  * @param w: tcp structure with destination address.
 152  * @param s: socket fd.
 153  * @return false on error, socket closed.
 154  */
 155 static int
 156 pick_outgoing_tcp(struct waiting_tcp* w, int s)
 157 {
 158         struct port_if* pi = NULL;
 159         int num;
 160 #ifdef INET6
 161         if(addr_is_ip6(&w->addr, w->addrlen))
 162                 num = w->outnet->num_ip6;
 163         else
 164 #endif
 165                 num = w->outnet->num_ip4;
 166         if(num == 0) {
 167                 log_err("no TCP outgoing interfaces of family");
 168                 log_addr(VERB_OPS, "for addr", &w->addr, w->addrlen);
 169                 sock_close(s);
 170                 return 0;
 171         }
 172 #ifdef INET6
 173         if(addr_is_ip6(&w->addr, w->addrlen))
 174                 pi = &w->outnet->ip6_ifs[ub_random_max(w->outnet->rnd, num)];
 175         else
 176 #endif
 177                 pi = &w->outnet->ip4_ifs[ub_random_max(w->outnet->rnd, num)];
 178         log_assert(pi);
 179         if(addr_is_any(&pi->addr, pi->addrlen)) {
 180                 /* binding to the ANY interface is for listening sockets */
 181                 return 1;
 182         }
 183         /* set port to 0 */
 184         if(addr_is_ip6(&pi->addr, pi->addrlen))
 185                 ((struct sockaddr_in6*)&pi->addr)->sin6_port = 0;
 186         else    ((struct sockaddr_in*)&pi->addr)->sin_port = 0;
 187         if(bind(s, (struct sockaddr*)&pi->addr, pi->addrlen) != 0) {
 188                 log_err("outgoing tcp: bind: %s", sock_strerror(errno));
 189                 sock_close(s);
 190                 return 0;
 191         }
 192         log_addr(VERB_ALGO, "tcp bound to src", &pi->addr, pi->addrlen);
 193         return 1;
 194 }
 195
 196 /** get TCP file descriptor for address, returns -1 on failure,
 197  * tcp_mss is 0 or maxseg size to set for TCP packets. */
 198 int
 199 outnet_get_tcp_fd(struct sockaddr_storage* addr, socklen_t addrlen, int tcp_mss, int dscp)
 200 {
 201         int s;
 202         int af;
 203         char* err;
 204 #ifdef SO_REUSEADDR
 205         int on = 1;
 206 #endif
 207 #ifdef INET6
 208         if(addr_is_ip6(addr, addrlen)){
 209                 s = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
 210                 af = AF_INET6;
 211         } else {
 212 #else
 213         {
 214 #endif
 215                 af = AF_INET;
 216                 s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 217         }
 218         if(s == -1) {
 219                 log_err_addr("outgoing tcp: socket", sock_strerror(errno),
 220                         addr, addrlen);
 221                 return -1;
 222         }
 223
 224 #ifdef SO_REUSEADDR
 225         if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on,
 226                 (socklen_t)sizeof(on)) < 0) {
 227                 verbose(VERB_ALGO, "outgoing tcp:"
 228                         " setsockopt(.. SO_REUSEADDR ..) failed");
 229         }
 230 #endif
 231
 232         err = set_ip_dscp(s, af, dscp);
 233         if(err != NULL) {
 234                 verbose(VERB_ALGO, "outgoing tcp:"
 235                         "error setting IP DiffServ codepoint on socket");
 236         }
 237
 238         if(tcp_mss > 0) {
 239 #if defined(IPPROTO_TCP) && defined(TCP_MAXSEG)
 240                 if(setsockopt(s, IPPROTO_TCP, TCP_MAXSEG,
 241                         (void*)&tcp_mss, (socklen_t)sizeof(tcp_mss)) < 0) {
 242                         verbose(VERB_ALGO, "outgoing tcp:"
 243                                 " setsockopt(.. TCP_MAXSEG ..) failed");
 244                 }
 245 #else
 246                 verbose(VERB_ALGO, "outgoing tcp:"
 247                         " setsockopt(TCP_MAXSEG) unsupported");
 248 #endif /* defined(IPPROTO_TCP) && defined(TCP_MAXSEG) */
 249         }
 250
 251         return s;
 252 }
 253
 254 /** connect tcp connection to addr, 0 on failure */
 255 int
 256 outnet_tcp_connect(int s, struct sockaddr_storage* addr, socklen_t addrlen)
 257 {
 258         if(connect(s, (struct sockaddr*)addr, addrlen) == -1) {
 259 #ifndef USE_WINSOCK
 260 #ifdef EINPROGRESS
 261                 if(errno != EINPROGRESS) {
 262 #endif
 263                         if(tcp_connect_errno_needs_log(
 264                                 (struct sockaddr*)addr, addrlen))
 265                                 log_err_addr("outgoing tcp: connect",
 266                                         strerror(errno), addr, addrlen);
 267                         close(s);
 268                         return 0;
 269 #ifdef EINPROGRESS
 270                 }
 271 #endif
 272 #else /* USE_WINSOCK */
 273                 if(WSAGetLastError() != WSAEINPROGRESS &&
 274                         WSAGetLastError() != WSAEWOULDBLOCK) {
 275                         closesocket(s);
 276                         return 0;
 277                 }
 278 #endif
 279         }
 280         return 1;
 281 }
 282
 283 /** use next free buffer to service a tcp query */
 284 static int
 285 outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 286 {
 287         struct pending_tcp* pend = w->outnet->tcp_free;
 288         int s;
 289         log_assert(pend);
 290         log_assert(pkt);
 291         log_assert(w->addrlen > 0);
 292         /* open socket */
 293         s = outnet_get_tcp_fd(&w->addr, w->addrlen, w->outnet->tcp_mss, w->outnet->ip_dscp);
 294
 295         if(s == -1)
 296                 return 0;
 297
 298         if(!pick_outgoing_tcp(w, s))
 299                 return 0;
 300
 301         fd_set_nonblock(s);
 302 #ifdef USE_OSX_MSG_FASTOPEN
 303         /* API for fast open is different here. We use a connectx() function and
 304            then writes can happen as normal even using SSL.*/
 305         /* connectx requires that the len be set in the sockaddr struct*/
 306         struct sockaddr_in *addr_in = (struct sockaddr_in *)&w->addr;
 307         addr_in->sin_len = w->addrlen;
 308         sa_endpoints_t endpoints;
 309         endpoints.sae_srcif = 0;
 310         endpoints.sae_srcaddr = NULL;
 311         endpoints.sae_srcaddrlen = 0;
 312         endpoints.sae_dstaddr = (struct sockaddr *)&w->addr;
 313         endpoints.sae_dstaddrlen = w->addrlen;
 314         if (connectx(s, &endpoints, SAE_ASSOCID_ANY,
 315                      CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE,
 316                      NULL, 0, NULL, NULL) == -1) {
 317                 /* if fails, failover to connect for OSX 10.10 */
 318 #ifdef EINPROGRESS
 319                 if(errno != EINPROGRESS) {
 320 #else
 321                 if(1) {
 322 #endif
 323                         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 324 #else /* USE_OSX_MSG_FASTOPEN*/
 325 #ifdef USE_MSG_FASTOPEN
 326         pend->c->tcp_do_fastopen = 1;
 327         /* Only do TFO for TCP in which case no connect() is required here.
 328            Don't combine client TFO with SSL, since OpenSSL can't
 329            currently support doing a handshake on fd that already isn't connected*/
 330         if (w->outnet->sslctx && w->ssl_upstream) {
 331                 if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 332 #else /* USE_MSG_FASTOPEN*/
 333         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 334 #endif /* USE_MSG_FASTOPEN*/
 335 #endif /* USE_OSX_MSG_FASTOPEN*/
 336 #ifndef USE_WINSOCK
 337 #ifdef EINPROGRESS
 338                 if(errno != EINPROGRESS) {
 339 #else
 340                 if(1) {
 341 #endif
 342                         if(tcp_connect_errno_needs_log(
 343                                 (struct sockaddr*)&w->addr, w->addrlen))
 344                                 log_err_addr("outgoing tcp: connect",
 345                                         strerror(errno), &w->addr, w->addrlen);
 346                         close(s);
 347 #else /* USE_WINSOCK */
 348                 if(WSAGetLastError() != WSAEINPROGRESS &&
 349                         WSAGetLastError() != WSAEWOULDBLOCK) {
 350                         closesocket(s);
 351 #endif
 352                         return 0;
 353                 }
 354         }
 355 #ifdef USE_MSG_FASTOPEN
 356         }
 357 #endif /* USE_MSG_FASTOPEN */
 358 #ifdef USE_OSX_MSG_FASTOPEN
 359                 }
 360         }
 361 #endif /* USE_OSX_MSG_FASTOPEN */
 362         if(w->outnet->sslctx && w->ssl_upstream) {
 363                 pend->c->ssl = outgoing_ssl_fd(w->outnet->sslctx, s);
 364                 if(!pend->c->ssl) {
 365                         pend->c->fd = s;
 366                         comm_point_close(pend->c);
 367                         return 0;
 368                 }
 369                 verbose(VERB_ALGO, "the query is using TLS encryption, for %s",
 370                         (w->tls_auth_name?w->tls_auth_name:"an unauthenticated connection"));
 371 #ifdef USE_WINSOCK
 372                 comm_point_tcp_win_bio_cb(pend->c, pend->c->ssl);
 373 #endif
 374                 pend->c->ssl_shake_state = comm_ssl_shake_write;
 375                 if(!set_auth_name_on_ssl(pend->c->ssl, w->tls_auth_name,
 376                         w->outnet->tls_use_sni)) {
 377                         pend->c->fd = s;
 378 #ifdef HAVE_SSL
 379                         SSL_free(pend->c->ssl);
 380 #endif
 381                         pend->c->ssl = NULL;
 382                         comm_point_close(pend->c);
 383                         return 0;
 384                 }
 385         }
 386         w->pkt = NULL;
 387         w->next_waiting = (void*)pend;
 388         pend->id = LDNS_ID_WIRE(pkt);
 389         w->outnet->num_tcp_outgoing++;
 390         w->outnet->tcp_free = pend->next_free;
 391         pend->next_free = NULL;
 392         pend->query = w;
 393         pend->c->repinfo.addrlen = w->addrlen;
 394         memcpy(&pend->c->repinfo.addr, &w->addr, w->addrlen);
 395         sldns_buffer_clear(pend->c->buffer);
 396         sldns_buffer_write(pend->c->buffer, pkt, pkt_len);
 397         sldns_buffer_flip(pend->c->buffer);
 398         pend->c->tcp_is_reading = 0;
 399         pend->c->tcp_byte_count = 0;
 400         comm_point_start_listening(pend->c, s, -1);
 401         return 1;
 402 }
 403
 404 /** see if buffers can be used to service TCP queries */
 405 static void
 406 use_free_buffer(struct outside_network* outnet)
 407 {
 408         struct waiting_tcp* w;
 409         while(outnet->tcp_free && outnet->tcp_wait_first
 410                 && !outnet->want_to_quit) {
 411                 w = outnet->tcp_wait_first;
 412                 outnet->tcp_wait_first = w->next_waiting;
 413                 if(outnet->tcp_wait_last == w)
 414                         outnet->tcp_wait_last = NULL;
 415                 if(!outnet_tcp_take_into_use(w, w->pkt, w->pkt_len)) {
 416                         comm_point_callback_type* cb = w->cb;
 417                         void* cb_arg = w->cb_arg;
 418                         waiting_tcp_delete(w);
 419                         fptr_ok(fptr_whitelist_pending_tcp(cb));
 420                         (void)(*cb)(NULL, cb_arg, NETEVENT_CLOSED, NULL);
 421                 }
 422         }
 423 }
 424
 425 /** decommission a tcp buffer, closes commpoint and frees waiting_tcp entry */
 426 static void
 427 decommission_pending_tcp(struct outside_network* outnet,
 428         struct pending_tcp* pend)
 429 {
 430         if(pend->c->ssl) {
 431 #ifdef HAVE_SSL
 432                 SSL_shutdown(pend->c->ssl);
 433                 SSL_free(pend->c->ssl);
 434                 pend->c->ssl = NULL;
 435 #endif
 436         }
 437         comm_point_close(pend->c);
 438         pend->next_free = outnet->tcp_free;
 439         outnet->tcp_free = pend;
 440         waiting_tcp_delete(pend->query);
 441         pend->query = NULL;
 442         use_free_buffer(outnet);
 443 }
 444
 445 int
 446 outnet_tcp_cb(struct comm_point* c, void* arg, int error,
 447         struct comm_reply *reply_info)
 448 {
 449         struct pending_tcp* pend = (struct pending_tcp*)arg;
 450         struct outside_network* outnet = pend->query->outnet;
 451         verbose(VERB_ALGO, "outnettcp cb");
 452         if(error != NETEVENT_NOERROR) {
 453                 verbose(VERB_QUERY, "outnettcp got tcp error %d", error);
 454                 /* pass error below and exit */
 455         } else {
 456                 /* check ID */
 457                 if(sldns_buffer_limit(c->buffer) < sizeof(uint16_t) ||
 458                         LDNS_ID_WIRE(sldns_buffer_begin(c->buffer))!=pend->id) {
 459                         log_addr(VERB_QUERY,
 460                                 "outnettcp: bad ID in reply, from:",
 461                                 &pend->query->addr, pend->query->addrlen);
 462                         error = NETEVENT_CLOSED;
 463                 }
 464         }
 465         fptr_ok(fptr_whitelist_pending_tcp(pend->query->cb));
 466         (void)(*pend->query->cb)(c, pend->query->cb_arg, error, reply_info);
 467         decommission_pending_tcp(outnet, pend);
 468         return 0;
 469 }
 470
 471 /** lower use count on pc, see if it can be closed */
 472 static void
 473 portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc)
 474 {
 475         struct port_if* pif;
 476         pc->num_outstanding--;
 477         if(pc->num_outstanding > 0) {
 478                 return;
 479         }
 480         /* close it and replace in unused list */
 481         verbose(VERB_ALGO, "close of port %d", pc->number);
 482         comm_point_close(pc->cp);
 483         pif = pc->pif;
 484         log_assert(pif->inuse > 0);
 485 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
 486         pif->avail_ports[pif->avail_total - pif->inuse] = pc->number;
 487 #endif
 488         pif->inuse--;
 489         pif->out[pc->index] = pif->out[pif->inuse];
 490         pif->out[pc->index]->index = pc->index;
 491         pc->next = outnet->unused_fds;
 492         outnet->unused_fds = pc;
 493 }
 494
 495 /** try to send waiting UDP queries */
 496 static void
 497 outnet_send_wait_udp(struct outside_network* outnet)
 498 {
 499         struct pending* pend;
 500         /* process waiting queries */
 501         while(outnet->udp_wait_first && outnet->unused_fds
 502                 && !outnet->want_to_quit) {
 503                 pend = outnet->udp_wait_first;
 504                 outnet->udp_wait_first = pend->next_waiting;
 505                 if(!pend->next_waiting) outnet->udp_wait_last = NULL;
 506                 sldns_buffer_clear(outnet->udp_buff);
 507                 sldns_buffer_write(outnet->udp_buff, pend->pkt, pend->pkt_len);
 508                 sldns_buffer_flip(outnet->udp_buff);
 509                 free(pend->pkt); /* freeing now makes get_mem correct */
 510                 pend->pkt = NULL;
 511                 pend->pkt_len = 0;
 512                 if(!randomize_and_send_udp(pend, outnet->udp_buff,
 513                         pend->timeout)) {
 514                         /* callback error on pending */
 515                         if(pend->cb) {
 516                                 fptr_ok(fptr_whitelist_pending_udp(pend->cb));
 517                                 (void)(*pend->cb)(outnet->unused_fds->cp, pend->cb_arg,
 518                                         NETEVENT_CLOSED, NULL);
 519                         }
 520                         pending_delete(outnet, pend);
 521                 }
 522         }
 523 }
 524
 525 int
 526 outnet_udp_cb(struct comm_point* c, void* arg, int error,
 527         struct comm_reply *reply_info)
 528 {
 529         struct outside_network* outnet = (struct outside_network*)arg;
 530         struct pending key;
 531         struct pending* p;
 532         verbose(VERB_ALGO, "answer cb");
 533
 534         if(error != NETEVENT_NOERROR) {
 535                 verbose(VERB_QUERY, "outnetudp got udp error %d", error);
 536                 return 0;
 537         }
 538         if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
 539                 verbose(VERB_QUERY, "outnetudp udp too short");
 540                 return 0;
 541         }
 542         log_assert(reply_info);
 543
 544         /* setup lookup key */
 545         key.id = (unsigned)LDNS_ID_WIRE(sldns_buffer_begin(c->buffer));
 546         memcpy(&key.addr, &reply_info->addr, reply_info->addrlen);
 547         key.addrlen = reply_info->addrlen;
 548         verbose(VERB_ALGO, "Incoming reply id = %4.4x", key.id);
 549         log_addr(VERB_ALGO, "Incoming reply addr =",
 550                 &reply_info->addr, reply_info->addrlen);
 551
 552         /* find it, see if this thing is a valid query response */
 553         verbose(VERB_ALGO, "lookup size is %d entries", (int)outnet->pending->count);
 554         p = (struct pending*)rbtree_search(outnet->pending, &key);
 555         if(!p) {
 556                 verbose(VERB_QUERY, "received unwanted or unsolicited udp reply dropped.");
 557                 log_buf(VERB_ALGO, "dropped message", c->buffer);
 558                 outnet->unwanted_replies++;
 559                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
 560                         >= outnet->unwanted_threshold) {
 561                         log_warn("unwanted reply total reached threshold (%u)"
 562                                 " you may be under attack."
 563                                 " defensive action: clearing the cache",
 564                                 (unsigned)outnet->unwanted_threshold);
 565                         fptr_ok(fptr_whitelist_alloc_cleanup(
 566                                 outnet->unwanted_action));
 567                         (*outnet->unwanted_action)(outnet->unwanted_param);
 568                         outnet->unwanted_total = 0;
 569                 }
 570                 return 0;
 571         }
 572
 573         verbose(VERB_ALGO, "received udp reply.");
 574         log_buf(VERB_ALGO, "udp message", c->buffer);
 575         if(p->pc->cp != c) {
 576                 verbose(VERB_QUERY, "received reply id,addr on wrong port. "
 577                         "dropped.");
 578                 outnet->unwanted_replies++;
 579                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
 580                         >= outnet->unwanted_threshold) {
 581                         log_warn("unwanted reply total reached threshold (%u)"
 582                                 " you may be under attack."
 583                                 " defensive action: clearing the cache",
 584                                 (unsigned)outnet->unwanted_threshold);
 585                         fptr_ok(fptr_whitelist_alloc_cleanup(
 586                                 outnet->unwanted_action));
 587                         (*outnet->unwanted_action)(outnet->unwanted_param);
 588                         outnet->unwanted_total = 0;
 589                 }
 590                 return 0;
 591         }
 592         comm_timer_disable(p->timer);
 593         verbose(VERB_ALGO, "outnet handle udp reply");
 594         /* delete from tree first in case callback creates a retry */
 595         (void)rbtree_delete(outnet->pending, p->node.key);
 596         if(p->cb) {
 597                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
 598                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_NOERROR, reply_info);
 599         }
 600         portcomm_loweruse(outnet, p->pc);
 601         pending_delete(NULL, p);
 602         outnet_send_wait_udp(outnet);
 603         return 0;
 604 }
 605
 606 /** calculate number of ip4 and ip6 interfaces*/
 607 static void
 608 calc_num46(char** ifs, int num_ifs, int do_ip4, int do_ip6,
 609         int* num_ip4, int* num_ip6)
 610 {
 611         int i;
 612         *num_ip4 = 0;
 613         *num_ip6 = 0;
 614         if(num_ifs <= 0) {
 615                 if(do_ip4)
 616                         *num_ip4 = 1;
 617                 if(do_ip6)
 618                         *num_ip6 = 1;
 619                 return;
 620         }
 621         for(i=0; i<num_ifs; i++)
 622         {
 623                 if(str_is_ip6(ifs[i])) {
 624                         if(do_ip6)
 625                                 (*num_ip6)++;
 626                 } else {
 627                         if(do_ip4)
 628                                 (*num_ip4)++;
 629                 }
 630         }
 631
 632 }
 633
 634 void
 635 pending_udp_timer_delay_cb(void* arg)
 636 {
 637         struct pending* p = (struct pending*)arg;
 638         struct outside_network* outnet = p->outnet;
 639         verbose(VERB_ALGO, "timeout udp with delay");
 640         portcomm_loweruse(outnet, p->pc);
 641         pending_delete(outnet, p);
 642         outnet_send_wait_udp(outnet);
 643 }
 644
 645 void
 646 pending_udp_timer_cb(void *arg)
 647 {
 648         struct pending* p = (struct pending*)arg;
 649         struct outside_network* outnet = p->outnet;
 650         /* it timed out */
 651         verbose(VERB_ALGO, "timeout udp");
 652         if(p->cb) {
 653                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
 654                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_TIMEOUT, NULL);
 655         }
 656         /* if delayclose, keep port open for a longer time.
 657          * But if the udpwaitlist exists, then we are struggling to
 658          * keep up with demand for sockets, so do not wait, but service
 659          * the customer (customer service more important than portICMPs) */
 660         if(outnet->delayclose && !outnet->udp_wait_first) {
 661                 p->cb = NULL;
 662                 p->timer->callback = &pending_udp_timer_delay_cb;
 663                 comm_timer_set(p->timer, &outnet->delay_tv);
 664                 return;
 665         }
 666         portcomm_loweruse(outnet, p->pc);
 667         pending_delete(outnet, p);
 668         outnet_send_wait_udp(outnet);
 669 }
 670
 671 /** create pending_tcp buffers */
 672 static int
 673 create_pending_tcp(struct outside_network* outnet, size_t bufsize)
 674 {
 675         size_t i;
 676         if(outnet->num_tcp == 0)
 677                 return 1; /* no tcp needed, nothing to do */
 678         if(!(outnet->tcp_conns = (struct pending_tcp **)calloc(
 679                         outnet->num_tcp, sizeof(struct pending_tcp*))))
 680                 return 0;
 681         for(i=0; i<outnet->num_tcp; i++) {
 682                 if(!(outnet->tcp_conns[i] = (struct pending_tcp*)calloc(1,
 683                         sizeof(struct pending_tcp))))
 684                         return 0;
 685                 outnet->tcp_conns[i]->next_free = outnet->tcp_free;
 686                 outnet->tcp_free = outnet->tcp_conns[i];
 687                 outnet->tcp_conns[i]->c = comm_point_create_tcp_out(
 688                         outnet->base, bufsize, outnet_tcp_cb,
 689                         outnet->tcp_conns[i]);
 690                 if(!outnet->tcp_conns[i]->c)
 691                         return 0;
 692         }
 693         return 1;
 694 }
 695
 696 /** setup an outgoing interface, ready address */
 697 static int setup_if(struct port_if* pif, const char* addrstr,
 698         int* avail, int numavail, size_t numfd)
 699 {
 700 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
 701         pif->avail_total = numavail;
 702         pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int));
 703         if(!pif->avail_ports)
 704                 return 0;
 705 #endif
 706         if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) &&
 707            !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT,
 708                               &pif->addr, &pif->addrlen, &pif->pfxlen))
 709                 return 0;
 710         pif->maxout = (int)numfd;
 711         pif->inuse = 0;
 712         pif->out = (struct port_comm**)calloc(numfd,
 713                 sizeof(struct port_comm*));
 714         if(!pif->out)
 715                 return 0;
 716         return 1;
 717 }
 718
 719 struct outside_network*
 720 outside_network_create(struct comm_base *base, size_t bufsize,
 721         size_t num_ports, char** ifs, int num_ifs, int do_ip4,
 722         int do_ip6, size_t num_tcp, int dscp, struct infra_cache* infra,
 723         struct ub_randstate* rnd, int use_caps_for_id, int* availports,
 724         int numavailports, size_t unwanted_threshold, int tcp_mss,
 725         void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
 726         void* sslctx, int delayclose, int tls_use_sni, struct dt_env* dtenv)
 727 {
 728         struct outside_network* outnet = (struct outside_network*)
 729                 calloc(1, sizeof(struct outside_network));
 730         size_t k;
 731         if(!outnet) {
 732                 log_err("malloc failed");
 733                 return NULL;
 734         }
 735         comm_base_timept(base, &outnet->now_secs, &outnet->now_tv);
 736         outnet->base = base;
 737         outnet->num_tcp = num_tcp;
 738         outnet->num_tcp_outgoing = 0;
 739         outnet->infra = infra;
 740         outnet->rnd = rnd;
 741         outnet->sslctx = sslctx;
 742         outnet->tls_use_sni = tls_use_sni;
 743 #ifdef USE_DNSTAP
 744         outnet->dtenv = dtenv;
 745 #else
 746         (void)dtenv;
 747 #endif
 748         outnet->svcd_overhead = 0;
 749         outnet->want_to_quit = 0;
 750         outnet->unwanted_threshold = unwanted_threshold;
 751         outnet->unwanted_action = unwanted_action;
 752         outnet->unwanted_param = unwanted_param;
 753         outnet->use_caps_for_id = use_caps_for_id;
 754         outnet->do_udp = do_udp;
 755         outnet->tcp_mss = tcp_mss;
 756         outnet->ip_dscp = dscp;
 757 #ifndef S_SPLINT_S
 758         if(delayclose) {
 759                 outnet->delayclose = 1;
 760                 outnet->delay_tv.tv_sec = delayclose/1000;
 761                 outnet->delay_tv.tv_usec = (delayclose%1000)*1000;
 762         }
 763 #endif
 764         if(numavailports == 0 || num_ports == 0) {
 765                 log_err("no outgoing ports available");
 766                 outside_network_delete(outnet);
 767                 return NULL;
 768         }
 769 #ifndef INET6
 770         do_ip6 = 0;
 771 #endif
 772         calc_num46(ifs, num_ifs, do_ip4, do_ip6,
 773                 &outnet->num_ip4, &outnet->num_ip6);
 774         if(outnet->num_ip4 != 0) {
 775                 if(!(outnet->ip4_ifs = (struct port_if*)calloc(
 776                         (size_t)outnet->num_ip4, sizeof(struct port_if)))) {
 777                         log_err("malloc failed");
 778                         outside_network_delete(outnet);
 779                         return NULL;
 780                 }
 781         }
 782         if(outnet->num_ip6 != 0) {
 783                 if(!(outnet->ip6_ifs = (struct port_if*)calloc(
 784                         (size_t)outnet->num_ip6, sizeof(struct port_if)))) {
 785                         log_err("malloc failed");
 786                         outside_network_delete(outnet);
 787                         return NULL;
 788                 }
 789         }
 790         if(     !(outnet->udp_buff = sldns_buffer_new(bufsize)) ||
 791                 !(outnet->pending = rbtree_create(pending_cmp)) ||
 792                 !(outnet->serviced = rbtree_create(serviced_cmp)) ||
 793                 !create_pending_tcp(outnet, bufsize)) {
 794                 log_err("malloc failed");
 795                 outside_network_delete(outnet);
 796                 return NULL;
 797         }
 798
 799         /* allocate commpoints */
 800         for(k=0; k<num_ports; k++) {
 801                 struct port_comm* pc;
 802                 pc = (struct port_comm*)calloc(1, sizeof(*pc));
 803                 if(!pc) {
 804                         log_err("malloc failed");
 805                         outside_network_delete(outnet);
 806                         return NULL;
 807                 }
 808                 pc->cp = comm_point_create_udp(outnet->base, -1,
 809                         outnet->udp_buff, outnet_udp_cb, outnet);
 810                 if(!pc->cp) {
 811                         log_err("malloc failed");
 812                         free(pc);
 813                         outside_network_delete(outnet);
 814                         return NULL;
 815                 }
 816                 pc->next = outnet->unused_fds;
 817                 outnet->unused_fds = pc;
 818         }
 819
 820         /* allocate interfaces */
 821         if(num_ifs == 0) {
 822                 if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0",
 823                         availports, numavailports, num_ports)) {
 824                         log_err("malloc failed");
 825                         outside_network_delete(outnet);
 826                         return NULL;
 827                 }
 828                 if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::",
 829                         availports, numavailports, num_ports)) {
 830                         log_err("malloc failed");
 831                         outside_network_delete(outnet);
 832                         return NULL;
 833                 }
 834         } else {
 835                 size_t done_4 = 0, done_6 = 0;
 836                 int i;
 837                 for(i=0; i<num_ifs; i++) {
 838                         if(str_is_ip6(ifs[i]) && do_ip6) {
 839                                 if(!setup_if(&outnet->ip6_ifs[done_6], ifs[i],
 840                                         availports, numavailports, num_ports)){
 841                                         log_err("malloc failed");
 842                                         outside_network_delete(outnet);
 843                                         return NULL;
 844                                 }
 845                                 done_6++;
 846                         }
 847                         if(!str_is_ip6(ifs[i]) && do_ip4) {
 848                                 if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i],
 849                                         availports, numavailports, num_ports)){
 850                                         log_err("malloc failed");
 851                                         outside_network_delete(outnet);
 852                                         return NULL;
 853                                 }
 854                                 done_4++;
 855                         }
 856                 }
 857         }
 858         return outnet;
 859 }
 860
 861 /** helper pending delete */
 862 static void
 863 pending_node_del(rbnode_type* node, void* arg)
 864 {
 865         struct pending* pend = (struct pending*)node;
 866         struct outside_network* outnet = (struct outside_network*)arg;
 867         pending_delete(outnet, pend);
 868 }
 869
 870 /** helper serviced delete */
 871 static void
 872 serviced_node_del(rbnode_type* node, void* ATTR_UNUSED(arg))
 873 {
 874         struct serviced_query* sq = (struct serviced_query*)node;
 875         struct service_callback* p = sq->cblist, *np;
 876         free(sq->qbuf);
 877         free(sq->zone);
 878         free(sq->tls_auth_name);
 879         edns_opt_list_free(sq->opt_list);
 880         while(p) {
 881                 np = p->next;
 882                 free(p);
 883                 p = np;
 884         }
 885         free(sq);
 886 }
 887
 888 void
 889 outside_network_quit_prepare(struct outside_network* outnet)
 890 {
 891         if(!outnet)
 892                 return;
 893         /* prevent queued items from being sent */
 894         outnet->want_to_quit = 1;
 895 }
 896
 897 void
 898 outside_network_delete(struct outside_network* outnet)
 899 {
 900         if(!outnet)
 901                 return;
 902         outnet->want_to_quit = 1;
 903         /* check every element, since we can be called on malloc error */
 904         if(outnet->pending) {
 905                 /* free pending elements, but do no unlink from tree. */
 906                 traverse_postorder(outnet->pending, pending_node_del, NULL);
 907                 free(outnet->pending);
 908         }
 909         if(outnet->serviced) {
 910                 traverse_postorder(outnet->serviced, serviced_node_del, NULL);
 911                 free(outnet->serviced);
 912         }
 913         if(outnet->udp_buff)
 914                 sldns_buffer_free(outnet->udp_buff);
 915         if(outnet->unused_fds) {
 916                 struct port_comm* p = outnet->unused_fds, *np;
 917                 while(p) {
 918                         np = p->next;
 919                         comm_point_delete(p->cp);
 920                         free(p);
 921                         p = np;
 922                 }
 923                 outnet->unused_fds = NULL;
 924         }
 925         if(outnet->ip4_ifs) {
 926                 int i, k;
 927                 for(i=0; i<outnet->num_ip4; i++) {
 928                         for(k=0; k<outnet->ip4_ifs[i].inuse; k++) {
 929                                 struct port_comm* pc = outnet->ip4_ifs[i].
 930                                         out[k];
 931                                 comm_point_delete(pc->cp);
 932                                 free(pc);
 933                         }
 934 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
 935                         free(outnet->ip4_ifs[i].avail_ports);
 936 #endif
 937                         free(outnet->ip4_ifs[i].out);
 938                 }
 939                 free(outnet->ip4_ifs);
 940         }
 941         if(outnet->ip6_ifs) {
 942                 int i, k;
 943                 for(i=0; i<outnet->num_ip6; i++) {
 944                         for(k=0; k<outnet->ip6_ifs[i].inuse; k++) {
 945                                 struct port_comm* pc = outnet->ip6_ifs[i].
 946                                         out[k];
 947                                 comm_point_delete(pc->cp);
 948                                 free(pc);
 949                         }
 950 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
 951                         free(outnet->ip6_ifs[i].avail_ports);
 952 #endif
 953                         free(outnet->ip6_ifs[i].out);
 954                 }
 955                 free(outnet->ip6_ifs);
 956         }
 957         if(outnet->tcp_conns) {
 958                 size_t i;
 959                 for(i=0; i<outnet->num_tcp; i++)
 960                         if(outnet->tcp_conns[i]) {
 961                                 comm_point_delete(outnet->tcp_conns[i]->c);
 962                                 waiting_tcp_delete(outnet->tcp_conns[i]->query);
 963                                 free(outnet->tcp_conns[i]);
 964                         }
 965                 free(outnet->tcp_conns);
 966         }
 967         if(outnet->tcp_wait_first) {
 968                 struct waiting_tcp* p = outnet->tcp_wait_first, *np;
 969                 while(p) {
 970                         np = p->next_waiting;
 971                         waiting_tcp_delete(p);
 972                         p = np;
 973                 }
 974         }
 975         if(outnet->udp_wait_first) {
 976                 struct pending* p = outnet->udp_wait_first, *np;
 977                 while(p) {
 978                         np = p->next_waiting;
 979                         pending_delete(NULL, p);
 980                         p = np;
 981                 }
 982         }
 983         free(outnet);
 984 }
 985
 986 void
 987 pending_delete(struct outside_network* outnet, struct pending* p)
 988 {
 989         if(!p)
 990                 return;
 991         if(outnet && outnet->udp_wait_first &&
 992                 (p->next_waiting || p == outnet->udp_wait_last) ) {
 993                 /* delete from waiting list, if it is in the waiting list */
 994                 struct pending* prev = NULL, *x = outnet->udp_wait_first;
 995                 while(x && x != p) {
 996                         prev = x;
 997                         x = x->next_waiting;
 998                 }
 999                 if(x) {
1000                         log_assert(x == p);
1001                         if(prev)
1002                                 prev->next_waiting = p->next_waiting;
1003                         else    outnet->udp_wait_first = p->next_waiting;
1004                         if(outnet->udp_wait_last == p)
1005                                 outnet->udp_wait_last = prev;
1006                 }
1007         }
1008         if(outnet) {
1009                 (void)rbtree_delete(outnet->pending, p->node.key);
1010         }
1011         if(p->timer)
1012                 comm_timer_delete(p->timer);
1013         free(p->pkt);
1014         free(p);
1015 }
1016
1017 static void
1018 sai6_putrandom(struct sockaddr_in6 *sa, int pfxlen, struct ub_randstate *rnd)
1019 {
1020         int i, last;
1021         if(!(pfxlen > 0 && pfxlen < 128))
1022                 return;
1023         for(i = 0; i < (128 - pfxlen) / 8; i++) {
1024                 sa->sin6_addr.s6_addr[15-i] = (uint8_t)ub_random_max(rnd, 256);
1025         }
1026         last = pfxlen & 7;
1027         if(last != 0) {
1028                 sa->sin6_addr.s6_addr[15-i] |=
1029                         ((0xFF >> last) & ub_random_max(rnd, 256));
1030         }
1031 }
1032
1033 /**
1034  * Try to open a UDP socket for outgoing communication.
1035  * Sets sockets options as needed.
1036  * @param addr: socket address.
1037  * @param addrlen: length of address.
1038  * @param pfxlen: length of network prefix (for address randomisation).
1039  * @param port: port override for addr.
1040  * @param inuse: if -1 is returned, this bool means the port was in use.
1041  * @param rnd: random state (for address randomisation).
1042  * @param dscp: DSCP to use.
1043  * @return fd or -1
1044  */
1045 static int
1046 udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen,
1047         int port, int* inuse, struct ub_randstate* rnd, int dscp)
1048 {
1049         int fd, noproto;
1050         if(addr_is_ip6(addr, addrlen)) {
1051                 int freebind = 0;
1052                 struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
1053                 sa.sin6_port = (in_port_t)htons((uint16_t)port);
1054                 sa.sin6_flowinfo = 0;
1055                 sa.sin6_scope_id = 0;
1056                 if(pfxlen != 0) {
1057                         freebind = 1;
1058                         sai6_putrandom(&sa, pfxlen, rnd);
1059                 }
1060                 fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
1061                         (struct sockaddr*)&sa, addrlen, 1, inuse, &noproto,
1062                         0, 0, 0, NULL, 0, freebind, 0, dscp);
1063         } else {
1064                 struct sockaddr_in* sa = (struct sockaddr_in*)addr;
1065                 sa->sin_port = (in_port_t)htons((uint16_t)port);
1066                 fd = create_udp_sock(AF_INET, SOCK_DGRAM,
1067                         (struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
1068                         0, 0, 0, NULL, 0, 0, 0, dscp);
1069         }
1070         return fd;
1071 }
1072
1073 /** Select random ID */
1074 static int
1075 select_id(struct outside_network* outnet, struct pending* pend,
1076         sldns_buffer* packet)
1077 {
1078         int id_tries = 0;
1079         pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
1080         LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
1081
1082         /* insert in tree */
1083         pend->node.key = pend;
1084         while(!rbtree_insert(outnet->pending, &pend->node)) {
1085                 /* change ID to avoid collision */
1086                 pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
1087                 LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
1088                 id_tries++;
1089                 if(id_tries == MAX_ID_RETRY) {
1090                         pend->id=99999; /* non existant ID */
1091                         log_err("failed to generate unique ID, drop msg");
1092                         return 0;
1093                 }
1094         }
1095         verbose(VERB_ALGO, "inserted new pending reply id=%4.4x", pend->id);
1096         return 1;
1097 }
1098
1099 /** Select random interface and port */
1100 static int
1101 select_ifport(struct outside_network* outnet, struct pending* pend,
1102         int num_if, struct port_if* ifs)
1103 {
1104         int my_if, my_port, fd, portno, inuse, tries=0;
1105         struct port_if* pif;
1106         /* randomly select interface and port */
1107         if(num_if == 0) {
1108                 verbose(VERB_QUERY, "Need to send query but have no "
1109                         "outgoing interfaces of that family");
1110                 return 0;
1111         }
1112         log_assert(outnet->unused_fds);
1113         tries = 0;
1114         while(1) {
1115                 my_if = ub_random_max(outnet->rnd, num_if);
1116                 pif = &ifs[my_if];
1117 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1118                 my_port = ub_random_max(outnet->rnd, pif->avail_total);
1119                 if(my_port < pif->inuse) {
1120                         /* port already open */
1121                         pend->pc = pif->out[my_port];
1122                         verbose(VERB_ALGO, "using UDP if=%d port=%d",
1123                                 my_if, pend->pc->number);
1124                         break;
1125                 }
1126                 /* try to open new port, if fails, loop to try again */
1127                 log_assert(pif->inuse < pif->maxout);
1128                 portno = pif->avail_ports[my_port - pif->inuse];
1129 #else
1130                 my_port = portno = 0;
1131 #endif
1132                 fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen,
1133                         portno, &inuse, outnet->rnd, outnet->ip_dscp);
1134                 if(fd == -1 && !inuse) {
1135                         /* nonrecoverable error making socket */
1136                         return 0;
1137                 }
1138                 if(fd != -1) {
1139                         verbose(VERB_ALGO, "opened UDP if=%d port=%d",
1140                                 my_if, portno);
1141                         /* grab fd */
1142                         pend->pc = outnet->unused_fds;
1143                         outnet->unused_fds = pend->pc->next;
1144
1145                         /* setup portcomm */
1146                         pend->pc->next = NULL;
1147                         pend->pc->number = portno;
1148                         pend->pc->pif = pif;
1149                         pend->pc->index = pif->inuse;
1150                         pend->pc->num_outstanding = 0;
1151                         comm_point_start_listening(pend->pc->cp, fd, -1);
1152
1153                         /* grab port in interface */
1154                         pif->out[pif->inuse] = pend->pc;
1155 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
1156                         pif->avail_ports[my_port - pif->inuse] =
1157                                 pif->avail_ports[pif->avail_total-pif->inuse-1];
1158 #endif
1159                         pif->inuse++;
1160                         break;
1161                 }
1162                 /* failed, already in use */
1163                 verbose(VERB_QUERY, "port %d in use, trying another", portno);
1164                 tries++;
1165                 if(tries == MAX_PORT_RETRY) {
1166                         log_err("failed to find an open port, drop msg");
1167                         return 0;
1168                 }
1169         }
1170         log_assert(pend->pc);
1171         pend->pc->num_outstanding++;
1172
1173         return 1;
1174 }
1175
1176 static int
1177 randomize_and_send_udp(struct pending* pend, sldns_buffer* packet, int timeout)
1178 {
1179         struct timeval tv;
1180         struct outside_network* outnet = pend->sq->outnet;
1181
1182         /* select id */
1183         if(!select_id(outnet, pend, packet)) {
1184                 return 0;
1185         }
1186
1187         /* select src_if, port */
1188         if(addr_is_ip6(&pend->addr, pend->addrlen)) {
1189                 if(!select_ifport(outnet, pend,
1190                         outnet->num_ip6, outnet->ip6_ifs))
1191                         return 0;
1192         } else {
1193                 if(!select_ifport(outnet, pend,
1194                         outnet->num_ip4, outnet->ip4_ifs))
1195                         return 0;
1196         }
1197         log_assert(pend->pc && pend->pc->cp);
1198
1199         /* send it over the commlink */
1200         if(!comm_point_send_udp_msg(pend->pc->cp, packet,
1201                 (struct sockaddr*)&pend->addr, pend->addrlen)) {
1202                 portcomm_loweruse(outnet, pend->pc);
1203                 return 0;
1204         }
1205
1206         /* system calls to set timeout after sending UDP to make roundtrip
1207            smaller. */
1208 #ifndef S_SPLINT_S
1209         tv.tv_sec = timeout/1000;
1210         tv.tv_usec = (timeout%1000)*1000;
1211 #endif
1212         comm_timer_set(pend->timer, &tv);
1213
1214 #ifdef USE_DNSTAP
1215         if(outnet->dtenv &&
1216            (outnet->dtenv->log_resolver_query_messages ||
1217             outnet->dtenv->log_forwarder_query_messages))
1218                 dt_msg_send_outside_query(outnet->dtenv, &pend->addr, comm_udp,
1219                 pend->sq->zone, pend->sq->zonelen, packet);
1220 #endif
1221         return 1;
1222 }
1223
1224 struct pending*
1225 pending_udp_query(struct serviced_query* sq, struct sldns_buffer* packet,
1226         int timeout, comm_point_callback_type* cb, void* cb_arg)
1227 {
1228         struct pending* pend = (struct pending*)calloc(1, sizeof(*pend));
1229         if(!pend) return NULL;
1230         pend->outnet = sq->outnet;
1231         pend->sq = sq;
1232         pend->addrlen = sq->addrlen;
1233         memmove(&pend->addr, &sq->addr, sq->addrlen);
1234         pend->cb = cb;
1235         pend->cb_arg = cb_arg;
1236         pend->node.key = pend;
1237         pend->timer = comm_timer_create(sq->outnet->base, pending_udp_timer_cb,
1238                 pend);
1239         if(!pend->timer) {
1240                 free(pend);
1241                 return NULL;
1242         }
1243
1244         if(sq->outnet->unused_fds == NULL) {
1245                 /* no unused fd, cannot create a new port (randomly) */
1246                 verbose(VERB_ALGO, "no fds available, udp query waiting");
1247                 pend->timeout = timeout;
1248                 pend->pkt_len = sldns_buffer_limit(packet);
1249                 pend->pkt = (uint8_t*)memdup(sldns_buffer_begin(packet),
1250                         pend->pkt_len);
1251                 if(!pend->pkt) {
1252                         comm_timer_delete(pend->timer);
1253                         free(pend);
1254                         return NULL;
1255                 }
1256                 /* put at end of waiting list */
1257                 if(sq->outnet->udp_wait_last)
1258                         sq->outnet->udp_wait_last->next_waiting = pend;
1259                 else
1260                         sq->outnet->udp_wait_first = pend;
1261                 sq->outnet->udp_wait_last = pend;
1262                 return pend;
1263         }
1264         if(!randomize_and_send_udp(pend, packet, timeout)) {
1265                 pending_delete(sq->outnet, pend);
1266                 return NULL;
1267         }
1268         return pend;
1269 }
1270
1271 void
1272 outnet_tcptimer(void* arg)
1273 {
1274         struct waiting_tcp* w = (struct waiting_tcp*)arg;
1275         struct outside_network* outnet = w->outnet;
1276         comm_point_callback_type* cb;
1277         void* cb_arg;
1278         if(w->pkt) {
1279                 /* it is on the waiting list */
1280                 waiting_list_remove(outnet, w);
1281         } else {
1282                 /* it was in use */
1283                 struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
1284                 if(pend->c->ssl) {
1285 #ifdef HAVE_SSL
1286                         SSL_shutdown(pend->c->ssl);
1287                         SSL_free(pend->c->ssl);
1288                         pend->c->ssl = NULL;
1289 #endif
1290                 }
1291                 comm_point_close(pend->c);
1292                 pend->query = NULL;
1293                 pend->next_free = outnet->tcp_free;
1294                 outnet->tcp_free = pend;
1295         }
1296         cb = w->cb;
1297         cb_arg = w->cb_arg;
1298         waiting_tcp_delete(w);
1299         fptr_ok(fptr_whitelist_pending_tcp(cb));
1300         (void)(*cb)(NULL, cb_arg, NETEVENT_TIMEOUT, NULL);
1301         use_free_buffer(outnet);
1302 }
1303
1304 struct waiting_tcp*
1305 pending_tcp_query(struct serviced_query* sq, sldns_buffer* packet,
1306         int timeout, comm_point_callback_type* callback, void* callback_arg)
1307 {
1308         struct pending_tcp* pend = sq->outnet->tcp_free;
1309         struct waiting_tcp* w;
1310         struct timeval tv;
1311         uint16_t id;
1312         /* if no buffer is free allocate space to store query */
1313         w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp)
1314                 + (pend?0:sldns_buffer_limit(packet)));
1315         if(!w) {
1316                 return NULL;
1317         }
1318         if(!(w->timer = comm_timer_create(sq->outnet->base, outnet_tcptimer, w))) {
1319                 free(w);
1320                 return NULL;
1321         }
1322         w->pkt = NULL;
1323         w->pkt_len = 0;
1324         id = ((unsigned)ub_random(sq->outnet->rnd)>>8) & 0xffff;
1325         LDNS_ID_SET(sldns_buffer_begin(packet), id);
1326         memcpy(&w->addr, &sq->addr, sq->addrlen);
1327         w->addrlen = sq->addrlen;
1328         w->outnet = sq->outnet;
1329         w->cb = callback;
1330         w->cb_arg = callback_arg;
1331         w->ssl_upstream = sq->ssl_upstream;
1332         w->tls_auth_name = sq->tls_auth_name;
1333 #ifndef S_SPLINT_S
1334         tv.tv_sec = timeout/1000;
1335         tv.tv_usec = (timeout%1000)*1000;
1336 #endif
1337         comm_timer_set(w->timer, &tv);
1338         if(pend) {
1339                 /* we have a buffer available right now */
1340                 if(!outnet_tcp_take_into_use(w, sldns_buffer_begin(packet),
1341                         sldns_buffer_limit(packet))) {
1342                         waiting_tcp_delete(w);
1343                         return NULL;
1344                 }
1345 #ifdef USE_DNSTAP
1346                 if(sq->outnet->dtenv &&
1347                    (sq->outnet->dtenv->log_resolver_query_messages ||
1348                     sq->outnet->dtenv->log_forwarder_query_messages))
1349                 dt_msg_send_outside_query(sq->outnet->dtenv, &sq->addr,
1350                 comm_tcp, sq->zone, sq->zonelen, packet);
1351 #endif
1352         } else {
1353                 /* queue up */
1354                 w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
1355                 w->pkt_len = sldns_buffer_limit(packet);
1356                 memmove(w->pkt, sldns_buffer_begin(packet), w->pkt_len);
1357                 w->next_waiting = NULL;
1358                 if(sq->outnet->tcp_wait_last)
1359                         sq->outnet->tcp_wait_last->next_waiting = w;
1360                 else    sq->outnet->tcp_wait_first = w;
1361                 sq->outnet->tcp_wait_last = w;
1362         }
1363         return w;
1364 }
1365
1366 /** create query for serviced queries */
1367 static void
1368 serviced_gen_query(sldns_buffer* buff, uint8_t* qname, size_t qnamelen,
1369         uint16_t qtype, uint16_t qclass, uint16_t flags)
1370 {
1371         sldns_buffer_clear(buff);
1372         /* skip id */
1373         sldns_buffer_write_u16(buff, flags);
1374         sldns_buffer_write_u16(buff, 1); /* qdcount */
1375         sldns_buffer_write_u16(buff, 0); /* ancount */
1376         sldns_buffer_write_u16(buff, 0); /* nscount */
1377         sldns_buffer_write_u16(buff, 0); /* arcount */
1378         sldns_buffer_write(buff, qname, qnamelen);
1379         sldns_buffer_write_u16(buff, qtype);
1380         sldns_buffer_write_u16(buff, qclass);
1381         sldns_buffer_flip(buff);
1382 }
1383
1384 /** lookup serviced query in serviced query rbtree */
1385 static struct serviced_query*
1386 lookup_serviced(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1387         struct sockaddr_storage* addr, socklen_t addrlen,
1388         struct edns_option* opt_list)
1389 {
1390         struct serviced_query key;
1391         key.node.key = &key;
1392         key.qbuf = sldns_buffer_begin(buff);
1393         key.qbuflen = sldns_buffer_limit(buff);
1394         key.dnssec = dnssec;
1395         memcpy(&key.addr, addr, addrlen);
1396         key.addrlen = addrlen;
1397         key.outnet = outnet;
1398         key.opt_list = opt_list;
1399         return (struct serviced_query*)rbtree_search(outnet->serviced, &key);
1400 }
1401
1402 /** Create new serviced entry */
1403 static struct serviced_query*
1404 serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1405         int want_dnssec, int nocaps, int tcp_upstream, int ssl_upstream,
1406         char* tls_auth_name, struct sockaddr_storage* addr, socklen_t addrlen,
1407         uint8_t* zone, size_t zonelen, int qtype, struct edns_option* opt_list)
1408 {
1409         struct serviced_query* sq = (struct serviced_query*)malloc(sizeof(*sq));
1410 #ifdef UNBOUND_DEBUG
1411         rbnode_type* ins;
1412 #endif
1413         if(!sq)
1414                 return NULL;
1415         sq->node.key = sq;
1416         sq->qbuf = memdup(sldns_buffer_begin(buff), sldns_buffer_limit(buff));
1417         if(!sq->qbuf) {
1418                 free(sq);
1419                 return NULL;
1420         }
1421         sq->qbuflen = sldns_buffer_limit(buff);
1422         sq->zone = memdup(zone, zonelen);
1423         if(!sq->zone) {
1424                 free(sq->qbuf);
1425                 free(sq);
1426                 return NULL;
1427         }
1428         sq->zonelen = zonelen;
1429         sq->qtype = qtype;
1430         sq->dnssec = dnssec;
1431         sq->want_dnssec = want_dnssec;
1432         sq->nocaps = nocaps;
1433         sq->tcp_upstream = tcp_upstream;
1434         sq->ssl_upstream = ssl_upstream;
1435         if(tls_auth_name) {
1436                 sq->tls_auth_name = strdup(tls_auth_name);
1437                 if(!sq->tls_auth_name) {
1438                         free(sq->zone);
1439                         free(sq->qbuf);
1440                         free(sq);
1441                         return NULL;
1442                 }
1443         } else {
1444                 sq->tls_auth_name = NULL;
1445         }
1446         memcpy(&sq->addr, addr, addrlen);
1447         sq->addrlen = addrlen;
1448         sq->opt_list = NULL;
1449         if(opt_list) {
1450                 sq->opt_list = edns_opt_copy_alloc(opt_list);
1451                 if(!sq->opt_list) {
1452                         free(sq->tls_auth_name);
1453                         free(sq->zone);
1454                         free(sq->qbuf);
1455                         free(sq);
1456                         return NULL;
1457                 }
1458         }
1459         sq->outnet = outnet;
1460         sq->cblist = NULL;
1461         sq->pending = NULL;
1462         sq->status = serviced_initial;
1463         sq->retry = 0;
1464         sq->to_be_deleted = 0;
1465 #ifdef UNBOUND_DEBUG
1466         ins =
1467 #else
1468         (void)
1469 #endif
1470         rbtree_insert(outnet->serviced, &sq->node);
1471         log_assert(ins != NULL); /* must not be already present */
1472         return sq;
1473 }
1474
1475 /** remove waiting tcp from the outnet waiting list */
1476 static void
1477 waiting_list_remove(struct outside_network* outnet, struct waiting_tcp* w)
1478 {
1479         struct waiting_tcp* p = outnet->tcp_wait_first, *prev = NULL;
1480         while(p) {
1481                 if(p == w) {
1482                         /* remove w */
1483                         if(prev)
1484                                 prev->next_waiting = w->next_waiting;
1485                         else    outnet->tcp_wait_first = w->next_waiting;
1486                         if(outnet->tcp_wait_last == w)
1487                                 outnet->tcp_wait_last = prev;
1488                         return;
1489                 }
1490                 prev = p;
1491                 p = p->next_waiting;
1492         }
1493 }
1494
1495 /** cleanup serviced query entry */
1496 static void
1497 serviced_delete(struct serviced_query* sq)
1498 {
1499         if(sq->pending) {
1500                 /* clear up the pending query */
1501                 if(sq->status == serviced_query_UDP_EDNS ||
1502                         sq->status == serviced_query_UDP ||
1503                         sq->status == serviced_query_UDP_EDNS_FRAG ||
1504                         sq->status == serviced_query_UDP_EDNS_fallback) {
1505                         struct pending* p = (struct pending*)sq->pending;
1506                         if(p->pc)
1507                                 portcomm_loweruse(sq->outnet, p->pc);
1508                         pending_delete(sq->outnet, p);
1509                         /* this call can cause reentrant calls back into the
1510                          * mesh */
1511                         outnet_send_wait_udp(sq->outnet);
1512                 } else {
1513                         struct waiting_tcp* p = (struct waiting_tcp*)
1514                                 sq->pending;
1515                         if(p->pkt == NULL) {
1516                                 decommission_pending_tcp(sq->outnet,
1517                                         (struct pending_tcp*)p->next_waiting);
1518                         } else {
1519                                 waiting_list_remove(sq->outnet, p);
1520                                 waiting_tcp_delete(p);
1521                         }
1522                 }
1523         }
1524         /* does not delete from tree, caller has to do that */
1525         serviced_node_del(&sq->node, NULL);
1526 }
1527
1528 /** perturb a dname capitalization randomly */
1529 static void
1530 serviced_perturb_qname(struct ub_randstate* rnd, uint8_t* qbuf, size_t len)
1531 {
1532         uint8_t lablen;
1533         uint8_t* d = qbuf + 10;
1534         long int random = 0;
1535         int bits = 0;
1536         log_assert(len >= 10 + 5 /* offset qname, root, qtype, qclass */);
1537         (void)len;
1538         lablen = *d++;
1539         while(lablen) {
1540                 while(lablen--) {
1541                         /* only perturb A-Z, a-z */
1542                         if(isalpha((unsigned char)*d)) {
1543                                 /* get a random bit */
1544                                 if(bits == 0) {
1545                                         random = ub_random(rnd);
1546                                         bits = 30;
1547                                 }
1548                                 if(random & 0x1) {
1549                                         *d = (uint8_t)toupper((unsigned char)*d);
1550                                 } else {
1551                                         *d = (uint8_t)tolower((unsigned char)*d);
1552                                 }
1553                                 random >>= 1;
1554                                 bits--;
1555                         }
1556                         d++;
1557                 }
1558                 lablen = *d++;
1559         }
1560         if(verbosity >= VERB_ALGO) {
1561                 char buf[LDNS_MAX_DOMAINLEN+1];
1562                 dname_str(qbuf+10, buf);
1563                 verbose(VERB_ALGO, "qname perturbed to %s", buf);
1564         }
1565 }
1566
1567 /** put serviced query into a buffer */
1568 static void
1569 serviced_encode(struct serviced_query* sq, sldns_buffer* buff, int with_edns)
1570 {
1571         /* if we are using 0x20 bits for ID randomness, perturb them */
1572         if(sq->outnet->use_caps_for_id && !sq->nocaps) {
1573                 serviced_perturb_qname(sq->outnet->rnd, sq->qbuf, sq->qbuflen);
1574         }
1575         /* generate query */
1576         sldns_buffer_clear(buff);
1577         sldns_buffer_write_u16(buff, 0); /* id placeholder */
1578         sldns_buffer_write(buff, sq->qbuf, sq->qbuflen);
1579         sldns_buffer_flip(buff);
1580         if(with_edns) {
1581                 /* add edns section */
1582                 struct edns_data edns;
1583                 edns.edns_present = 1;
1584                 edns.ext_rcode = 0;
1585                 edns.edns_version = EDNS_ADVERTISED_VERSION;
1586                 edns.opt_list = sq->opt_list;
1587                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1588                         if(addr_is_ip6(&sq->addr, sq->addrlen)) {
1589                                 if(EDNS_FRAG_SIZE_IP6 < EDNS_ADVERTISED_SIZE)
1590                                         edns.udp_size = EDNS_FRAG_SIZE_IP6;
1591                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
1592                         } else {
1593                                 if(EDNS_FRAG_SIZE_IP4 < EDNS_ADVERTISED_SIZE)
1594                                         edns.udp_size = EDNS_FRAG_SIZE_IP4;
1595                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
1596                         }
1597                 } else {
1598                         edns.udp_size = EDNS_ADVERTISED_SIZE;
1599                 }
1600                 edns.bits = 0;
1601                 if(sq->dnssec & EDNS_DO)
1602                         edns.bits = EDNS_DO;
1603                 if(sq->dnssec & BIT_CD)
1604                         LDNS_CD_SET(sldns_buffer_begin(buff));
1605                 attach_edns_record(buff, &edns);
1606         }
1607 }
1608
1609 /**
1610  * Perform serviced query UDP sending operation.
1611  * Sends UDP with EDNS, unless infra host marked non EDNS.
1612  * @param sq: query to send.
1613  * @param buff: buffer scratch space.
1614  * @return 0 on error.
1615  */
1616 static int
1617 serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
1618 {
1619         int rtt, vs;
1620         uint8_t edns_lame_known;
1621         time_t now = *sq->outnet->now_secs;
1622
1623         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1624                 sq->zonelen, now, &vs, &edns_lame_known, &rtt))
1625                 return 0;
1626         sq->last_rtt = rtt;
1627         verbose(VERB_ALGO, "EDNS lookup known=%d vs=%d", edns_lame_known, vs);
1628         if(sq->status == serviced_initial) {
1629                 if(vs != -1) {
1630                         sq->status = serviced_query_UDP_EDNS;
1631                 } else {
1632                         sq->status = serviced_query_UDP;
1633                 }
1634         }
1635         serviced_encode(sq, buff, (sq->status == serviced_query_UDP_EDNS) ||
1636                 (sq->status == serviced_query_UDP_EDNS_FRAG));
1637         sq->last_sent_time = *sq->outnet->now_tv;
1638         sq->edns_lame_known = (int)edns_lame_known;
1639         verbose(VERB_ALGO, "serviced query UDP timeout=%d msec", rtt);
1640         sq->pending = pending_udp_query(sq, buff, rtt,
1641                 serviced_udp_callback, sq);
1642         if(!sq->pending)
1643                 return 0;
1644         return 1;
1645 }
1646
1647 /** check that perturbed qname is identical */
1648 static int
1649 serviced_check_qname(sldns_buffer* pkt, uint8_t* qbuf, size_t qbuflen)
1650 {
1651         uint8_t* d1 = sldns_buffer_begin(pkt)+12;
1652         uint8_t* d2 = qbuf+10;
1653         uint8_t len1, len2;
1654         int count = 0;
1655         if(sldns_buffer_limit(pkt) < 12+1+4) /* packet too small for qname */
1656                 return 0;
1657         log_assert(qbuflen >= 15 /* 10 header, root, type, class */);
1658         len1 = *d1++;
1659         len2 = *d2++;
1660         while(len1 != 0 || len2 != 0) {
1661                 if(LABEL_IS_PTR(len1)) {
1662                         /* check if we can read *d1 with compression ptr rest */
1663                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1664                                 return 0;
1665                         d1 = sldns_buffer_begin(pkt)+PTR_OFFSET(len1, *d1);
1666                         /* check if we can read the destination *d1 */
1667                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1668                                 return 0;
1669                         len1 = *d1++;
1670                         if(count++ > MAX_COMPRESS_PTRS)
1671                                 return 0;
1672                         continue;
1673                 }
1674                 if(d2 > qbuf+qbuflen)
1675                         return 0;
1676                 if(len1 != len2)
1677                         return 0;
1678                 if(len1 > LDNS_MAX_LABELLEN)
1679                         return 0;
1680                 /* check len1 + 1(next length) are okay to read */
1681                 if(d1+len1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1682                         return 0;
1683                 log_assert(len1 <= LDNS_MAX_LABELLEN);
1684                 log_assert(len2 <= LDNS_MAX_LABELLEN);
1685                 log_assert(len1 == len2 && len1 != 0);
1686                 /* compare the labels - bitwise identical */
1687                 if(memcmp(d1, d2, len1) != 0)
1688                         return 0;
1689                 d1 += len1;
1690                 d2 += len2;
1691                 len1 = *d1++;
1692                 len2 = *d2++;
1693         }
1694         return 1;
1695 }
1696
1697 /** call the callbacks for a serviced query */
1698 static void
1699 serviced_callbacks(struct serviced_query* sq, int error, struct comm_point* c,
1700         struct comm_reply* rep)
1701 {
1702         struct service_callback* p;
1703         int dobackup = (sq->cblist && sq->cblist->next); /* >1 cb*/
1704         uint8_t *backup_p = NULL;
1705         size_t backlen = 0;
1706 #ifdef UNBOUND_DEBUG
1707         rbnode_type* rem =
1708 #else
1709         (void)
1710 #endif
1711         /* remove from tree, and schedule for deletion, so that callbacks
1712          * can safely deregister themselves and even create new serviced
1713          * queries that are identical to this one. */
1714         rbtree_delete(sq->outnet->serviced, sq);
1715         log_assert(rem); /* should have been present */
1716         sq->to_be_deleted = 1;
1717         verbose(VERB_ALGO, "svcd callbacks start");
1718         if(sq->outnet->use_caps_for_id && error == NETEVENT_NOERROR && c &&
1719                 !sq->nocaps && sq->qtype != LDNS_RR_TYPE_PTR) {
1720                 /* for type PTR do not check perturbed name in answer,
1721                  * compatibility with cisco dns guard boxes that mess up
1722                  * reverse queries 0x20 contents */
1723                 /* noerror and nxdomain must have a qname in reply */
1724                 if(sldns_buffer_read_u16_at(c->buffer, 4) == 0 &&
1725                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1726                                 == LDNS_RCODE_NOERROR ||
1727                          LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1728                                 == LDNS_RCODE_NXDOMAIN)) {
1729                         verbose(VERB_DETAIL, "no qname in reply to check 0x20ID");
1730                         log_addr(VERB_DETAIL, "from server",
1731                                 &sq->addr, sq->addrlen);
1732                         log_buf(VERB_DETAIL, "for packet", c->buffer);
1733                         error = NETEVENT_CLOSED;
1734                         c = NULL;
1735                 } else if(sldns_buffer_read_u16_at(c->buffer, 4) > 0 &&
1736                         !serviced_check_qname(c->buffer, sq->qbuf,
1737                         sq->qbuflen)) {
1738                         verbose(VERB_DETAIL, "wrong 0x20-ID in reply qname");
1739                         log_addr(VERB_DETAIL, "from server",
1740                                 &sq->addr, sq->addrlen);
1741                         log_buf(VERB_DETAIL, "for packet", c->buffer);
1742                         error = NETEVENT_CAPSFAIL;
1743                         /* and cleanup too */
1744                         pkt_dname_tolower(c->buffer,
1745                                 sldns_buffer_at(c->buffer, 12));
1746                 } else {
1747                         verbose(VERB_ALGO, "good 0x20-ID in reply qname");
1748                         /* cleanup caps, prettier cache contents. */
1749                         pkt_dname_tolower(c->buffer,
1750                                 sldns_buffer_at(c->buffer, 12));
1751                 }
1752         }
1753         if(dobackup && c) {
1754                 /* make a backup of the query, since the querystate processing
1755                  * may send outgoing queries that overwrite the buffer.
1756                  * use secondary buffer to store the query.
1757                  * This is a data copy, but faster than packet to server */
1758                 backlen = sldns_buffer_limit(c->buffer);
1759                 backup_p = memdup(sldns_buffer_begin(c->buffer), backlen);
1760                 if(!backup_p) {
1761                         log_err("malloc failure in serviced query callbacks");
1762                         error = NETEVENT_CLOSED;
1763                         c = NULL;
1764                 }
1765                 sq->outnet->svcd_overhead = backlen;
1766         }
1767         /* test the actual sq->cblist, because the next elem could be deleted*/
1768         while((p=sq->cblist) != NULL) {
1769                 sq->cblist = p->next; /* remove this element */
1770                 if(dobackup && c) {
1771                         sldns_buffer_clear(c->buffer);
1772                         sldns_buffer_write(c->buffer, backup_p, backlen);
1773                         sldns_buffer_flip(c->buffer);
1774                 }
1775                 fptr_ok(fptr_whitelist_serviced_query(p->cb));
1776                 (void)(*p->cb)(c, p->cb_arg, error, rep);
1777                 free(p);
1778         }
1779         if(backup_p) {
1780                 free(backup_p);
1781                 sq->outnet->svcd_overhead = 0;
1782         }
1783         verbose(VERB_ALGO, "svcd callbacks end");
1784         log_assert(sq->cblist == NULL);
1785         serviced_delete(sq);
1786 }
1787
1788 int
1789 serviced_tcp_callback(struct comm_point* c, void* arg, int error,
1790         struct comm_reply* rep)
1791 {
1792         struct serviced_query* sq = (struct serviced_query*)arg;
1793         struct comm_reply r2;
1794         sq->pending = NULL; /* removed after this callback */
1795         if(error != NETEVENT_NOERROR)
1796                 log_addr(VERB_QUERY, "tcp error for address",
1797                         &sq->addr, sq->addrlen);
1798         if(error==NETEVENT_NOERROR)
1799                 infra_update_tcp_works(sq->outnet->infra, &sq->addr,
1800                         sq->addrlen, sq->zone, sq->zonelen);
1801 #ifdef USE_DNSTAP
1802         if(error==NETEVENT_NOERROR && sq->outnet->dtenv &&
1803            (sq->outnet->dtenv->log_resolver_response_messages ||
1804             sq->outnet->dtenv->log_forwarder_response_messages))
1805                 dt_msg_send_outside_response(sq->outnet->dtenv, &sq->addr,
1806                 c->type, sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1807                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1808 #endif
1809         if(error==NETEVENT_NOERROR && sq->status == serviced_query_TCP_EDNS &&
1810                 (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1811                 LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(sldns_buffer_begin(
1812                 c->buffer)) == LDNS_RCODE_NOTIMPL) ) {
1813                 /* attempt to fallback to nonEDNS */
1814                 sq->status = serviced_query_TCP_EDNS_fallback;
1815                 serviced_tcp_initiate(sq, c->buffer);
1816                 return 0;
1817         } else if(error==NETEVENT_NOERROR &&
1818                 sq->status == serviced_query_TCP_EDNS_fallback &&
1819                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1820                         LDNS_RCODE_NOERROR || LDNS_RCODE_WIRE(
1821                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NXDOMAIN
1822                         || LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1823                         == LDNS_RCODE_YXDOMAIN)) {
1824                 /* the fallback produced a result that looks promising, note
1825                  * that this server should be approached without EDNS */
1826                 /* only store noEDNS in cache if domain is noDNSSEC */
1827                 if(!sq->want_dnssec)
1828                   if(!infra_edns_update(sq->outnet->infra, &sq->addr,
1829                         sq->addrlen, sq->zone, sq->zonelen, -1,
1830                         *sq->outnet->now_secs))
1831                         log_err("Out of memory caching no edns for host");
1832                 sq->status = serviced_query_TCP;
1833         }
1834         if(sq->tcp_upstream || sq->ssl_upstream) {
1835             struct timeval now = *sq->outnet->now_tv;
1836             if(error!=NETEVENT_NOERROR) {
1837                 if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
1838                     sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
1839                     -1, sq->last_rtt, (time_t)now.tv_sec))
1840                     log_err("out of memory in TCP exponential backoff.");
1841             } else if(now.tv_sec > sq->last_sent_time.tv_sec ||
1842                 (now.tv_sec == sq->last_sent_time.tv_sec &&
1843                 now.tv_usec > sq->last_sent_time.tv_usec)) {
1844                 /* convert from microseconds to milliseconds */
1845                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1846                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1847                 verbose(VERB_ALGO, "measured TCP-time at %d msec", roundtime);
1848                 log_assert(roundtime >= 0);
1849                 /* only store if less then AUTH_TIMEOUT seconds, it could be
1850                  * huge due to system-hibernated and we woke up */
1851                 if(roundtime < 60000) {
1852                     if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
1853                         sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
1854                         roundtime, sq->last_rtt, (time_t)now.tv_sec))
1855                         log_err("out of memory noting rtt.");
1856                 }
1857             }
1858         }
1859         /* insert address into reply info */
1860         if(!rep) {
1861                 /* create one if there isn't (on errors) */
1862                 rep = &r2;
1863                 r2.c = c;
1864         }
1865         memcpy(&rep->addr, &sq->addr, sq->addrlen);
1866         rep->addrlen = sq->addrlen;
1867         serviced_callbacks(sq, error, c, rep);
1868         return 0;
1869 }
1870
1871 static void
1872 serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
1873 {
1874         verbose(VERB_ALGO, "initiate TCP query %s",
1875                 sq->status==serviced_query_TCP_EDNS?"EDNS":"");
1876         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1877         sq->last_sent_time = *sq->outnet->now_tv;
1878         sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1879                 serviced_tcp_callback, sq);
1880         if(!sq->pending) {
1881                 /* delete from tree so that a retry by above layer does not
1882                  * clash with this entry */
1883                 verbose(VERB_ALGO, "serviced_tcp_initiate: failed to send tcp query");
1884                 serviced_callbacks(sq, NETEVENT_CLOSED, NULL, NULL);
1885         }
1886 }
1887
1888 /** Send serviced query over TCP return false on initial failure */
1889 static int
1890 serviced_tcp_send(struct serviced_query* sq, sldns_buffer* buff)
1891 {
1892         int vs, rtt, timeout;
1893         uint8_t edns_lame_known;
1894         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1895                 sq->zonelen, *sq->outnet->now_secs, &vs, &edns_lame_known,
1896                 &rtt))
1897                 return 0;
1898         sq->last_rtt = rtt;
1899         if(vs != -1)
1900                 sq->status = serviced_query_TCP_EDNS;
1901         else    sq->status = serviced_query_TCP;
1902         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1903         sq->last_sent_time = *sq->outnet->now_tv;
1904         if(sq->tcp_upstream || sq->ssl_upstream) {
1905                 timeout = rtt;
1906                 if(rtt >= UNKNOWN_SERVER_NICENESS && rtt < TCP_AUTH_QUERY_TIMEOUT)
1907                         timeout = TCP_AUTH_QUERY_TIMEOUT;
1908         } else {
1909                 timeout = TCP_AUTH_QUERY_TIMEOUT;
1910         }
1911         sq->pending = pending_tcp_query(sq, buff, timeout,
1912                 serviced_tcp_callback, sq);
1913         return sq->pending != NULL;
1914 }
1915
1916 /* see if packet is edns malformed; got zeroes at start.
1917  * This is from servers that return malformed packets to EDNS0 queries,
1918  * but they return good packets for nonEDNS0 queries.
1919  * We try to detect their output; without resorting to a full parse or
1920  * check for too many bytes after the end of the packet. */
1921 static int
1922 packet_edns_malformed(struct sldns_buffer* buf, int qtype)
1923 {
1924         size_t len;
1925         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE)
1926                 return 1; /* malformed */
1927         /* they have NOERROR rcode, 1 answer. */
1928         if(LDNS_RCODE_WIRE(sldns_buffer_begin(buf)) != LDNS_RCODE_NOERROR)
1929                 return 0;
1930         /* one query (to skip) and answer records */
1931         if(LDNS_QDCOUNT(sldns_buffer_begin(buf)) != 1 ||
1932                 LDNS_ANCOUNT(sldns_buffer_begin(buf)) == 0)
1933                 return 0;
1934         /* skip qname */
1935         len = dname_valid(sldns_buffer_at(buf, LDNS_HEADER_SIZE),
1936                 sldns_buffer_limit(buf)-LDNS_HEADER_SIZE);
1937         if(len == 0)
1938                 return 0;
1939         if(len == 1 && qtype == 0)
1940                 return 0; /* we asked for '.' and type 0 */
1941         /* and then 4 bytes (type and class of query) */
1942         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE + len + 4 + 3)
1943                 return 0;
1944
1945         /* and start with 11 zeroes as the answer RR */
1946         /* so check the qtype of the answer record, qname=0, type=0 */
1947         if(sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[0] == 0 &&
1948            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[1] == 0 &&
1949            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[2] == 0)
1950                 return 1;
1951         return 0;
1952 }
1953
1954 int
1955 serviced_udp_callback(struct comm_point* c, void* arg, int error,
1956         struct comm_reply* rep)
1957 {
1958         struct serviced_query* sq = (struct serviced_query*)arg;
1959         struct outside_network* outnet = sq->outnet;
1960         struct timeval now = *sq->outnet->now_tv;
1961
1962         sq->pending = NULL; /* removed after callback */
1963         if(error == NETEVENT_TIMEOUT) {
1964                 if(sq->status == serviced_query_UDP_EDNS && sq->last_rtt < 5000) {
1965                         /* fallback to 1480/1280 */
1966                         sq->status = serviced_query_UDP_EDNS_FRAG;
1967                         log_name_addr(VERB_ALGO, "try edns1xx0", sq->qbuf+10,
1968                                 &sq->addr, sq->addrlen);
1969                         if(!serviced_udp_send(sq, c->buffer)) {
1970                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1971                         }
1972                         return 0;
1973                 }
1974                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1975                         /* fragmentation size did not fix it */
1976                         sq->status = serviced_query_UDP_EDNS;
1977                 }
1978                 sq->retry++;
1979                 if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1980                         sq->zone, sq->zonelen, sq->qtype, -1, sq->last_rtt,
1981                         (time_t)now.tv_sec))
1982                         log_err("out of memory in UDP exponential backoff");
1983                 if(sq->retry < OUTBOUND_UDP_RETRY) {
1984                         log_name_addr(VERB_ALGO, "retry query", sq->qbuf+10,
1985                                 &sq->addr, sq->addrlen);
1986                         if(!serviced_udp_send(sq, c->buffer)) {
1987                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1988                         }
1989                         return 0;
1990                 }
1991         }
1992         if(error != NETEVENT_NOERROR) {
1993                 /* udp returns error (due to no ID or interface available) */
1994                 serviced_callbacks(sq, error, c, rep);
1995                 return 0;
1996         }
1997 #ifdef USE_DNSTAP
1998         if(error == NETEVENT_NOERROR && outnet->dtenv &&
1999            (outnet->dtenv->log_resolver_response_messages ||
2000             outnet->dtenv->log_forwarder_response_messages))
2001                 dt_msg_send_outside_response(outnet->dtenv, &sq->addr, c->type,
2002                 sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
2003                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
2004 #endif
2005         if( (sq->status == serviced_query_UDP_EDNS
2006                 ||sq->status == serviced_query_UDP_EDNS_FRAG)
2007                 && (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
2008                         == LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(
2009                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOTIMPL
2010                     || packet_edns_malformed(c->buffer, sq->qtype)
2011                         )) {
2012                 /* try to get an answer by falling back without EDNS */
2013                 verbose(VERB_ALGO, "serviced query: attempt without EDNS");
2014                 sq->status = serviced_query_UDP_EDNS_fallback;
2015                 sq->retry = 0;
2016                 if(!serviced_udp_send(sq, c->buffer)) {
2017                         serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
2018                 }
2019                 return 0;
2020         } else if(sq->status == serviced_query_UDP_EDNS &&
2021                 !sq->edns_lame_known) {
2022                 /* now we know that edns queries received answers store that */
2023                 log_addr(VERB_ALGO, "serviced query: EDNS works for",
2024                         &sq->addr, sq->addrlen);
2025                 if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
2026                         sq->zone, sq->zonelen, 0, (time_t)now.tv_sec)) {
2027                         log_err("Out of memory caching edns works");
2028                 }
2029                 sq->edns_lame_known = 1;
2030         } else if(sq->status == serviced_query_UDP_EDNS_fallback &&
2031                 !sq->edns_lame_known && (LDNS_RCODE_WIRE(
2032                 sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOERROR ||
2033                 LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
2034                 LDNS_RCODE_NXDOMAIN || LDNS_RCODE_WIRE(sldns_buffer_begin(
2035                 c->buffer)) == LDNS_RCODE_YXDOMAIN)) {
2036                 /* the fallback produced a result that looks promising, note
2037                  * that this server should be approached without EDNS */
2038                 /* only store noEDNS in cache if domain is noDNSSEC */
2039                 if(!sq->want_dnssec) {
2040                   log_addr(VERB_ALGO, "serviced query: EDNS fails for",
2041                         &sq->addr, sq->addrlen);
2042                   if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
2043                         sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
2044                         log_err("Out of memory caching no edns for host");
2045                   }
2046                 } else {
2047                   log_addr(VERB_ALGO, "serviced query: EDNS fails, but "
2048                         "not stored because need DNSSEC for", &sq->addr,
2049                         sq->addrlen);
2050                 }
2051                 sq->status = serviced_query_UDP;
2052         }
2053         if(now.tv_sec > sq->last_sent_time.tv_sec ||
2054                 (now.tv_sec == sq->last_sent_time.tv_sec &&
2055                 now.tv_usec > sq->last_sent_time.tv_usec)) {
2056                 /* convert from microseconds to milliseconds */
2057                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
2058                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
2059                 verbose(VERB_ALGO, "measured roundtrip at %d msec", roundtime);
2060                 log_assert(roundtime >= 0);
2061                 /* in case the system hibernated, do not enter a huge value,
2062                  * above this value gives trouble with server selection */
2063                 if(roundtime < 60000) {
2064                     if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
2065                         sq->zone, sq->zonelen, sq->qtype, roundtime,
2066                         sq->last_rtt, (time_t)now.tv_sec))
2067                         log_err("out of memory noting rtt.");
2068                 }
2069         }
2070         /* perform TC flag check and TCP fallback after updating our
2071          * cache entries for EDNS status and RTT times */
2072         if(LDNS_TC_WIRE(sldns_buffer_begin(c->buffer))) {
2073                 /* fallback to TCP */
2074                 /* this discards partial UDP contents */
2075                 if(sq->status == serviced_query_UDP_EDNS ||
2076                         sq->status == serviced_query_UDP_EDNS_FRAG ||
2077                         sq->status == serviced_query_UDP_EDNS_fallback)
2078                         /* if we have unfinished EDNS_fallback, start again */
2079                         sq->status = serviced_query_TCP_EDNS;
2080                 else    sq->status = serviced_query_TCP;
2081                 serviced_tcp_initiate(sq, c->buffer);
2082                 return 0;
2083         }
2084         /* yay! an answer */
2085         serviced_callbacks(sq, error, c, rep);
2086         return 0;
2087 }
2088
2089 struct serviced_query*
2090 outnet_serviced_query(struct outside_network* outnet,
2091         struct query_info* qinfo, uint16_t flags, int dnssec, int want_dnssec,
2092         int nocaps, int tcp_upstream, int ssl_upstream, char* tls_auth_name,
2093         struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
2094         size_t zonelen, struct module_qstate* qstate,
2095         comm_point_callback_type* callback, void* callback_arg, sldns_buffer* buff,
2096         struct module_env* env)
2097 {
2098         struct serviced_query* sq;
2099         struct service_callback* cb;
2100         struct edns_tag_addr* client_tag_addr;
2101
2102         if(!inplace_cb_query_call(env, qinfo, flags, addr, addrlen, zone, zonelen,
2103                 qstate, qstate->region))
2104                         return NULL;
2105
2106         if((client_tag_addr = edns_tag_addr_lookup(&env->edns_tags->client_tags,
2107                 addr, addrlen))) {
2108                 uint16_t client_tag = htons(client_tag_addr->tag_data);
2109                 edns_opt_list_append(&qstate->edns_opts_back_out,
2110                         env->edns_tags->client_tag_opcode, 2,
2111                         (uint8_t*)&client_tag, qstate->region);
2112         }
2113
2114         serviced_gen_query(buff, qinfo->qname, qinfo->qname_len, qinfo->qtype,
2115                 qinfo->qclass, flags);
2116         sq = lookup_serviced(outnet, buff, dnssec, addr, addrlen,
2117                 qstate->edns_opts_back_out);
2118         /* duplicate entries are included in the callback list, because
2119          * there is a counterpart registration by our caller that needs to
2120          * be doubly-removed (with callbacks perhaps). */
2121         if(!(cb = (struct service_callback*)malloc(sizeof(*cb))))
2122                 return NULL;
2123         if(!sq) {
2124                 /* make new serviced query entry */
2125                 sq = serviced_create(outnet, buff, dnssec, want_dnssec, nocaps,
2126                         tcp_upstream, ssl_upstream, tls_auth_name, addr,
2127                         addrlen, zone, zonelen, (int)qinfo->qtype,
2128                         qstate->edns_opts_back_out);
2129                 if(!sq) {
2130                         free(cb);
2131                         return NULL;
2132                 }
2133                 /* perform first network action */
2134                 if(outnet->do_udp && !(tcp_upstream || ssl_upstream)) {
2135                         if(!serviced_udp_send(sq, buff)) {
2136                                 (void)rbtree_delete(outnet->serviced, sq);
2137                                 serviced_node_del(&sq->node, NULL);
2138                                 free(cb);
2139                                 return NULL;
2140                         }
2141                 } else {
2142                         if(!serviced_tcp_send(sq, buff)) {
2143                                 (void)rbtree_delete(outnet->serviced, sq);
2144                                 serviced_node_del(&sq->node, NULL);
2145                                 free(cb);
2146                                 return NULL;
2147                         }
2148                 }
2149         }
2150         /* add callback to list of callbacks */
2151         cb->cb = callback;
2152         cb->cb_arg = callback_arg;
2153         cb->next = sq->cblist;
2154         sq->cblist = cb;
2155         return sq;
2156 }
2157
2158 /** remove callback from list */
2159 static void
2160 callback_list_remove(struct serviced_query* sq, void* cb_arg)
2161 {
2162         struct service_callback** pp = &sq->cblist;
2163         while(*pp) {
2164                 if((*pp)->cb_arg == cb_arg) {
2165                         struct service_callback* del = *pp;
2166                         *pp = del->next;
2167                         free(del);
2168                         return;
2169                 }
2170                 pp = &(*pp)->next;
2171         }
2172 }
2173
2174 void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
2175 {
2176         if(!sq)
2177                 return;
2178         callback_list_remove(sq, cb_arg);
2179         /* if callbacks() routine scheduled deletion, let it do that */
2180         if(!sq->cblist && !sq->to_be_deleted) {
2181                 (void)rbtree_delete(sq->outnet->serviced, sq);
2182                 serviced_delete(sq);
2183         }
2184 }
2185
2186 /** create fd to send to this destination */
2187 static int
2188 fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr,
2189         socklen_t to_addrlen)
2190 {
2191         struct sockaddr_storage* addr;
2192         socklen_t addrlen;
2193         int i, try, pnum, dscp;
2194         struct port_if* pif;
2195
2196         /* create fd */
2197         dscp = outnet->ip_dscp;
2198         for(try = 0; try<1000; try++) {
2199                 int port = 0;
2200                 int freebind = 0;
2201                 int noproto = 0;
2202                 int inuse = 0;
2203                 int fd = -1;
2204
2205                 /* select interface */
2206                 if(addr_is_ip6(to_addr, to_addrlen)) {
2207                         if(outnet->num_ip6 == 0) {
2208                                 char to[64];
2209                                 addr_to_str(to_addr, to_addrlen, to, sizeof(to));
2210                                 verbose(VERB_QUERY, "need ipv6 to send, but no ipv6 outgoing interfaces, for %s", to);
2211                                 return -1;
2212                         }
2213                         i = ub_random_max(outnet->rnd, outnet->num_ip6);
2214                         pif = &outnet->ip6_ifs[i];
2215                 } else {
2216                         if(outnet->num_ip4 == 0) {
2217                                 char to[64];
2218                                 addr_to_str(to_addr, to_addrlen, to, sizeof(to));
2219                                 verbose(VERB_QUERY, "need ipv4 to send, but no ipv4 outgoing interfaces, for %s", to);
2220                                 return -1;
2221                         }
2222                         i = ub_random_max(outnet->rnd, outnet->num_ip4);
2223                         pif = &outnet->ip4_ifs[i];
2224                 }
2225                 addr = &pif->addr;
2226                 addrlen = pif->addrlen;
2227 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
2228                 pnum = ub_random_max(outnet->rnd, pif->avail_total);
2229                 if(pnum < pif->inuse) {
2230                         /* port already open */
2231                         port = pif->out[pnum]->number;
2232                 } else {
2233                         /* unused ports in start part of array */
2234                         port = pif->avail_ports[pnum - pif->inuse];
2235                 }
2236 #else
2237                 pnum = port = 0;
2238 #endif
2239                 if(addr_is_ip6(to_addr, to_addrlen)) {
2240                         struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
2241                         sa.sin6_port = (in_port_t)htons((uint16_t)port);
2242                         fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
2243                                 (struct sockaddr*)&sa, addrlen, 1, &inuse, &noproto,
2244                                 0, 0, 0, NULL, 0, freebind, 0, dscp);
2245                 } else {
2246                         struct sockaddr_in* sa = (struct sockaddr_in*)addr;
2247                         sa->sin_port = (in_port_t)htons((uint16_t)port);
2248                         fd = create_udp_sock(AF_INET, SOCK_DGRAM,
2249                                 (struct sockaddr*)addr, addrlen, 1, &inuse, &noproto,
2250                                 0, 0, 0, NULL, 0, freebind, 0, dscp);
2251                 }
2252                 if(fd != -1) {
2253                         return fd;
2254                 }
2255                 if(!inuse) {
2256                         return -1;
2257                 }
2258         }
2259         /* too many tries */
2260         log_err("cannot send probe, ports are in use");
2261         return -1;
2262 }
2263
2264 struct comm_point*
2265 outnet_comm_point_for_udp(struct outside_network* outnet,
2266         comm_point_callback_type* cb, void* cb_arg,
2267         struct sockaddr_storage* to_addr, socklen_t to_addrlen)
2268 {
2269         struct comm_point* cp;
2270         int fd = fd_for_dest(outnet, to_addr, to_addrlen);
2271         if(fd == -1) {
2272                 return NULL;
2273         }
2274         cp = comm_point_create_udp(outnet->base, fd, outnet->udp_buff,
2275                 cb, cb_arg);
2276         if(!cp) {
2277                 log_err("malloc failure");
2278                 close(fd);
2279                 return NULL;
2280         }
2281         return cp;
2282 }
2283
2284 /** setup SSL for comm point */
2285 static int
2286 setup_comm_ssl(struct comm_point* cp, struct outside_network* outnet,
2287         int fd, char* host)
2288 {
2289         cp->ssl = outgoing_ssl_fd(outnet->sslctx, fd);
2290         if(!cp->ssl) {
2291                 log_err("cannot create SSL object");
2292                 return 0;
2293         }
2294 #ifdef USE_WINSOCK
2295         comm_point_tcp_win_bio_cb(cp, cp->ssl);
2296 #endif
2297         cp->ssl_shake_state = comm_ssl_shake_write;
2298         /* https verification */
2299 #ifdef HAVE_SSL
2300         if(outnet->tls_use_sni) {
2301                 (void)SSL_set_tlsext_host_name(cp->ssl, host);
2302         }
2303 #endif
2304 #ifdef HAVE_SSL_SET1_HOST
2305         if((SSL_CTX_get_verify_mode(outnet->sslctx)&SSL_VERIFY_PEER)) {
2306                 /* because we set SSL_VERIFY_PEER, in netevent in
2307                  * ssl_handshake, it'll check if the certificate
2308                  * verification has succeeded */
2309                 /* SSL_VERIFY_PEER is set on the sslctx */
2310                 /* and the certificates to verify with are loaded into
2311                  * it with SSL_load_verify_locations or
2312                  * SSL_CTX_set_default_verify_paths */
2313                 /* setting the hostname makes openssl verify the
2314                  * host name in the x509 certificate in the
2315                  * SSL connection*/
2316                 if(!SSL_set1_host(cp->ssl, host)) {
2317                         log_err("SSL_set1_host failed");
2318                         return 0;
2319                 }
2320         }
2321 #elif defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
2322         /* openssl 1.0.2 has this function that can be used for
2323          * set1_host like verification */
2324         if((SSL_CTX_get_verify_mode(outnet->sslctx)&SSL_VERIFY_PEER)) {
2325                 X509_VERIFY_PARAM* param = SSL_get0_param(cp->ssl);
2326 #  ifdef X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS
2327                 X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
2328 #  endif
2329                 if(!X509_VERIFY_PARAM_set1_host(param, host, strlen(host))) {
2330                         log_err("X509_VERIFY_PARAM_set1_host failed");
2331                         return 0;
2332                 }
2333         }
2334 #else
2335         (void)host;
2336 #endif /* HAVE_SSL_SET1_HOST */
2337         return 1;
2338 }
2339
2340 struct comm_point*
2341 outnet_comm_point_for_tcp(struct outside_network* outnet,
2342         comm_point_callback_type* cb, void* cb_arg,
2343         struct sockaddr_storage* to_addr, socklen_t to_addrlen,
2344         sldns_buffer* query, int timeout, int ssl, char* host)
2345 {
2346         struct comm_point* cp;
2347         int fd = outnet_get_tcp_fd(to_addr, to_addrlen, outnet->tcp_mss, outnet->ip_dscp);
2348         if(fd == -1) {
2349                 return 0;
2350         }
2351         fd_set_nonblock(fd);
2352         if(!outnet_tcp_connect(fd, to_addr, to_addrlen)) {
2353                 /* outnet_tcp_connect has closed fd on error for us */
2354                 return 0;
2355         }
2356         cp = comm_point_create_tcp_out(outnet->base, 65552, cb, cb_arg);
2357         if(!cp) {
2358                 log_err("malloc failure");
2359                 close(fd);
2360                 return 0;
2361         }
2362         cp->repinfo.addrlen = to_addrlen;
2363         memcpy(&cp->repinfo.addr, to_addr, to_addrlen);
2364
2365         /* setup for SSL (if needed) */
2366         if(ssl) {
2367                 if(!setup_comm_ssl(cp, outnet, fd, host)) {
2368                         log_err("cannot setup XoT");
2369                         comm_point_delete(cp);
2370                         return NULL;
2371                 }
2372         }
2373
2374         /* set timeout on TCP connection */
2375         comm_point_start_listening(cp, fd, timeout);
2376         /* copy scratch buffer to cp->buffer */
2377         sldns_buffer_copy(cp->buffer, query);
2378         return cp;
2379 }
2380
2381 /** setup http request headers in buffer for sending query to destination */
2382 static int
2383 setup_http_request(sldns_buffer* buf, char* host, char* path)
2384 {
2385         sldns_buffer_clear(buf);
2386         sldns_buffer_printf(buf, "GET /%s HTTP/1.1\r\n", path);
2387         sldns_buffer_printf(buf, "Host: %s\r\n", host);
2388         sldns_buffer_printf(buf, "User-Agent: unbound/%s\r\n",
2389                 PACKAGE_VERSION);
2390         /* We do not really do multiple queries per connection,
2391          * but this header setting is also not needed.
2392          * sldns_buffer_printf(buf, "Connection: close\r\n") */
2393         sldns_buffer_printf(buf, "\r\n");
2394         if(sldns_buffer_position(buf)+10 > sldns_buffer_capacity(buf))
2395                 return 0; /* somehow buffer too short, but it is about 60K
2396                 and the request is only a couple bytes long. */
2397         sldns_buffer_flip(buf);
2398         return 1;
2399 }
2400
2401 struct comm_point*
2402 outnet_comm_point_for_http(struct outside_network* outnet,
2403         comm_point_callback_type* cb, void* cb_arg,
2404         struct sockaddr_storage* to_addr, socklen_t to_addrlen, int timeout,
2405         int ssl, char* host, char* path)
2406 {
2407         /* cp calls cb with err=NETEVENT_DONE when transfer is done */
2408         struct comm_point* cp;
2409         int fd = outnet_get_tcp_fd(to_addr, to_addrlen, outnet->tcp_mss, outnet->ip_dscp);
2410         if(fd == -1) {
2411                 return 0;
2412         }
2413         fd_set_nonblock(fd);
2414         if(!outnet_tcp_connect(fd, to_addr, to_addrlen)) {
2415                 /* outnet_tcp_connect has closed fd on error for us */
2416                 return 0;
2417         }
2418         cp = comm_point_create_http_out(outnet->base, 65552, cb, cb_arg,
2419                 outnet->udp_buff);
2420         if(!cp) {
2421                 log_err("malloc failure");
2422                 close(fd);
2423                 return 0;
2424         }
2425         cp->repinfo.addrlen = to_addrlen;
2426         memcpy(&cp->repinfo.addr, to_addr, to_addrlen);
2427
2428         /* setup for SSL (if needed) */
2429         if(ssl) {
2430                 if(!setup_comm_ssl(cp, outnet, fd, host)) {
2431                         log_err("cannot setup https");
2432                         comm_point_delete(cp);
2433                         return NULL;
2434                 }
2435         }
2436
2437         /* set timeout on TCP connection */
2438         comm_point_start_listening(cp, fd, timeout);
2439
2440         /* setup http request in cp->buffer */
2441         if(!setup_http_request(cp->buffer, host, path)) {
2442                 log_err("error setting up http request");
2443                 comm_point_delete(cp);
2444                 return NULL;
2445         }
2446         return cp;
2447 }
2448
2449 /** get memory used by waiting tcp entry (in use or not) */
2450 static size_t
2451 waiting_tcp_get_mem(struct waiting_tcp* w)
2452 {
2453         size_t s;
2454         if(!w) return 0;
2455         s = sizeof(*w) + w->pkt_len;
2456         if(w->timer)
2457                 s += comm_timer_get_mem(w->timer);
2458         return s;
2459 }
2460
2461 /** get memory used by port if */
2462 static size_t
2463 if_get_mem(struct port_if* pif)
2464 {
2465         size_t s;
2466         int i;
2467         s = sizeof(*pif) +
2468 #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION
2469             sizeof(int)*pif->avail_total +
2470 #endif
2471                 sizeof(struct port_comm*)*pif->maxout;
2472         for(i=0; i<pif->inuse; i++)
2473                 s += sizeof(*pif->out[i]) +
2474                         comm_point_get_mem(pif->out[i]->cp);
2475         return s;
2476 }
2477
2478 /** get memory used by waiting udp */
2479 static size_t
2480 waiting_udp_get_mem(struct pending* w)
2481 {
2482         size_t s;
2483         s = sizeof(*w) + comm_timer_get_mem(w->timer) + w->pkt_len;
2484         return s;
2485 }
2486
2487 size_t outnet_get_mem(struct outside_network* outnet)
2488 {
2489         size_t i;
2490         int k;
2491         struct waiting_tcp* w;
2492         struct pending* u;
2493         struct serviced_query* sq;
2494         struct service_callback* sb;
2495         struct port_comm* pc;
2496         size_t s = sizeof(*outnet) + sizeof(*outnet->base) +
2497                 sizeof(*outnet->udp_buff) +
2498                 sldns_buffer_capacity(outnet->udp_buff);
2499         /* second buffer is not ours */
2500         for(pc = outnet->unused_fds; pc; pc = pc->next) {
2501                 s += sizeof(*pc) + comm_point_get_mem(pc->cp);
2502         }
2503         for(k=0; k<outnet->num_ip4; k++)
2504                 s += if_get_mem(&outnet->ip4_ifs[k]);
2505         for(k=0; k<outnet->num_ip6; k++)
2506                 s += if_get_mem(&outnet->ip6_ifs[k]);
2507         for(u=outnet->udp_wait_first; u; u=u->next_waiting)
2508                 s += waiting_udp_get_mem(u);
2509
2510         s += sizeof(struct pending_tcp*)*outnet->num_tcp;
2511         for(i=0; i<outnet->num_tcp; i++) {
2512                 s += sizeof(struct pending_tcp);
2513                 s += comm_point_get_mem(outnet->tcp_conns[i]->c);
2514                 if(outnet->tcp_conns[i]->query)
2515                         s += waiting_tcp_get_mem(outnet->tcp_conns[i]->query);
2516         }
2517         for(w=outnet->tcp_wait_first; w; w = w->next_waiting)
2518                 s += waiting_tcp_get_mem(w);
2519         s += sizeof(*outnet->pending);
2520         s += (sizeof(struct pending) + comm_timer_get_mem(NULL)) *
2521                 outnet->pending->count;
2522         s += sizeof(*outnet->serviced);
2523         s += outnet->svcd_overhead;
2524         RBTREE_FOR(sq, struct serviced_query*, outnet->serviced) {
2525                 s += sizeof(*sq) + sq->qbuflen;
2526                 for(sb = sq->cblist; sb; sb = sb->next)
2527                         s += sizeof(*sb);
2528         }
2529         return s;
2530 }
2531
2532 size_t
2533 serviced_get_mem(struct serviced_query* sq)
2534 {
2535         struct service_callback* sb;
2536         size_t s;
2537         s = sizeof(*sq) + sq->qbuflen;
2538         for(sb = sq->cblist; sb; sb = sb->next)
2539                 s += sizeof(*sb);
2540         if(sq->status == serviced_query_UDP_EDNS ||
2541                 sq->status == serviced_query_UDP ||
2542                 sq->status == serviced_query_UDP_EDNS_FRAG ||
2543                 sq->status == serviced_query_UDP_EDNS_fallback) {
2544                 s += sizeof(struct pending);
2545                 s += comm_timer_get_mem(NULL);
2546         } else {
2547                 /* does not have size of the pkt pointer */
2548                 /* always has a timer except on malloc failures */
2549
2550                 /* these sizes are part of the main outside network mem */
2551                 /*
2552                 s += sizeof(struct waiting_tcp);
2553                 s += comm_timer_get_mem(NULL);
2554                 */
2555         }
2556         return s;
2557 }
2558