services/outside_network.c

   1 /*
   2  * services/outside_network.c - implement sending of queries and wait answer.
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file has functions to send queries to authoritative servers and
  40  * wait for the pending answer events.
  41  */
  42 #include "config.h"
  43 #include <ctype.h>
  44 #ifdef HAVE_SYS_TYPES_H
  45 #  include <sys/types.h>
  46 #endif
  47 #include <sys/time.h>
  48 #include "services/outside_network.h"
  49 #include "services/listen_dnsport.h"
  50 #include "services/cache/infra.h"
  51 #include "util/data/msgparse.h"
  52 #include "util/data/msgreply.h"
  53 #include "util/data/msgencode.h"
  54 #include "util/data/dname.h"
  55 #include "util/netevent.h"
  56 #include "util/log.h"
  57 #include "util/net_help.h"
  58 #include "util/random.h"
  59 #include "util/fptr_wlist.h"
  60 #include "sldns/sbuffer.h"
  61 #include "dnstap/dnstap.h"
  62 #ifdef HAVE_OPENSSL_SSL_H
  63 #include <openssl/ssl.h>
  64 #endif
  65
  66 #ifdef HAVE_NETDB_H
  67 #include <netdb.h>
  68 #endif
  69 #include <fcntl.h>
  70
  71 /** number of times to retry making a random ID that is unique. */
  72 #define MAX_ID_RETRY 1000
  73 /** number of times to retry finding interface, port that can be opened. */
  74 #define MAX_PORT_RETRY 10000
  75 /** number of retries on outgoing UDP queries */
  76 #define OUTBOUND_UDP_RETRY 1
  77
  78 /** initiate TCP transaction for serviced query */
  79 static void serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff);
  80 /** with a fd available, randomize and send UDP */
  81 static int randomize_and_send_udp(struct pending* pend, sldns_buffer* packet,
  82         int timeout);
  83
  84 /** remove waiting tcp from the outnet waiting list */
  85 static void waiting_list_remove(struct outside_network* outnet,
  86         struct waiting_tcp* w);
  87
  88 int
  89 pending_cmp(const void* key1, const void* key2)
  90 {
  91         struct pending *p1 = (struct pending*)key1;
  92         struct pending *p2 = (struct pending*)key2;
  93         if(p1->id < p2->id)
  94                 return -1;
  95         if(p1->id > p2->id)
  96                 return 1;
  97         log_assert(p1->id == p2->id);
  98         return sockaddr_cmp(&p1->addr, p1->addrlen, &p2->addr, p2->addrlen);
  99 }
 100
 101 int
 102 serviced_cmp(const void* key1, const void* key2)
 103 {
 104         struct serviced_query* q1 = (struct serviced_query*)key1;
 105         struct serviced_query* q2 = (struct serviced_query*)key2;
 106         int r;
 107         if(q1->qbuflen < q2->qbuflen)
 108                 return -1;
 109         if(q1->qbuflen > q2->qbuflen)
 110                 return 1;
 111         log_assert(q1->qbuflen == q2->qbuflen);
 112         log_assert(q1->qbuflen >= 15 /* 10 header, root, type, class */);
 113         /* alternate casing of qname is still the same query */
 114         if((r = memcmp(q1->qbuf, q2->qbuf, 10)) != 0)
 115                 return r;
 116         if((r = memcmp(q1->qbuf+q1->qbuflen-4, q2->qbuf+q2->qbuflen-4, 4)) != 0)
 117                 return r;
 118         if(q1->dnssec != q2->dnssec) {
 119                 if(q1->dnssec < q2->dnssec)
 120                         return -1;
 121                 return 1;
 122         }
 123         if((r = query_dname_compare(q1->qbuf+10, q2->qbuf+10)) != 0)
 124                 return r;
 125         if((r = edns_opt_list_compare(q1->opt_list, q2->opt_list)) != 0)
 126                 return r;
 127         return sockaddr_cmp(&q1->addr, q1->addrlen, &q2->addr, q2->addrlen);
 128 }
 129
 130 /** delete waiting_tcp entry. Does not unlink from waiting list.
 131  * @param w: to delete.
 132  */
 133 static void
 134 waiting_tcp_delete(struct waiting_tcp* w)
 135 {
 136         if(!w) return;
 137         if(w->timer)
 138                 comm_timer_delete(w->timer);
 139         free(w);
 140 }
 141
 142 /**
 143  * Pick random outgoing-interface of that family, and bind it.
 144  * port set to 0 so OS picks a port number for us.
 145  * if it is the ANY address, do not bind.
 146  * @param w: tcp structure with destination address.
 147  * @param s: socket fd.
 148  * @return false on error, socket closed.
 149  */
 150 static int
 151 pick_outgoing_tcp(struct waiting_tcp* w, int s)
 152 {
 153         struct port_if* pi = NULL;
 154         int num;
 155 #ifdef INET6
 156         if(addr_is_ip6(&w->addr, w->addrlen))
 157                 num = w->outnet->num_ip6;
 158         else
 159 #endif
 160                 num = w->outnet->num_ip4;
 161         if(num == 0) {
 162                 log_err("no TCP outgoing interfaces of family");
 163                 log_addr(VERB_OPS, "for addr", &w->addr, w->addrlen);
 164 #ifndef USE_WINSOCK
 165                 close(s);
 166 #else
 167                 closesocket(s);
 168 #endif
 169                 return 0;
 170         }
 171 #ifdef INET6
 172         if(addr_is_ip6(&w->addr, w->addrlen))
 173                 pi = &w->outnet->ip6_ifs[ub_random_max(w->outnet->rnd, num)];
 174         else
 175 #endif
 176                 pi = &w->outnet->ip4_ifs[ub_random_max(w->outnet->rnd, num)];
 177         log_assert(pi);
 178         if(addr_is_any(&pi->addr, pi->addrlen)) {
 179                 /* binding to the ANY interface is for listening sockets */
 180                 return 1;
 181         }
 182         /* set port to 0 */
 183         if(addr_is_ip6(&pi->addr, pi->addrlen))
 184                 ((struct sockaddr_in6*)&pi->addr)->sin6_port = 0;
 185         else    ((struct sockaddr_in*)&pi->addr)->sin_port = 0;
 186         if(bind(s, (struct sockaddr*)&pi->addr, pi->addrlen) != 0) {
 187 #ifndef USE_WINSOCK
 188                 log_err("outgoing tcp: bind: %s", strerror(errno));
 189                 close(s);
 190 #else
 191                 log_err("outgoing tcp: bind: %s",
 192                         wsa_strerror(WSAGetLastError()));
 193                 closesocket(s);
 194 #endif
 195                 return 0;
 196         }
 197         log_addr(VERB_ALGO, "tcp bound to src", &pi->addr, pi->addrlen);
 198         return 1;
 199 }
 200
 201 /** use next free buffer to service a tcp query */
 202 static int
 203 outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 204 {
 205         struct pending_tcp* pend = w->outnet->tcp_free;
 206         int s;
 207         log_assert(pend);
 208         log_assert(pkt);
 209         log_assert(w->addrlen > 0);
 210         /* open socket */
 211 #ifdef INET6
 212         if(addr_is_ip6(&w->addr, w->addrlen))
 213                 s = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
 214         else
 215 #endif
 216                 s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 217         if(s == -1) {
 218 #ifndef USE_WINSOCK
 219                 log_err_addr("outgoing tcp: socket", strerror(errno),
 220                         &w->addr, w->addrlen);
 221 #else
 222                 log_err_addr("outgoing tcp: socket",
 223                         wsa_strerror(WSAGetLastError()), &w->addr, w->addrlen);
 224 #endif
 225                 return 0;
 226         }
 227
 228         if (w->outnet->tcp_mss > 0) {
 229 #if defined(IPPROTO_TCP) && defined(TCP_MAXSEG)
 230                 if(setsockopt(s, IPPROTO_TCP, TCP_MAXSEG,
 231                         (void*)&w->outnet->tcp_mss,
 232                         (socklen_t)sizeof(w->outnet->tcp_mss)) < 0) {
 233                         verbose(VERB_ALGO, "outgoing tcp:"
 234                                 " setsockopt(.. SO_REUSEADDR ..) failed");
 235                 }
 236 #else
 237                 verbose(VERB_ALGO, "outgoing tcp:"
 238                         " setsockopt(TCP_MAXSEG) unsupported");
 239 #endif /* defined(IPPROTO_TCP) && defined(TCP_MAXSEG) */
 240         }
 241
 242         if(!pick_outgoing_tcp(w, s))
 243                 return 0;
 244
 245         fd_set_nonblock(s);
 246 #ifdef USE_OSX_MSG_FASTOPEN
 247         /* API for fast open is different here. We use a connectx() function and
 248            then writes can happen as normal even using SSL.*/
 249         /* connectx requires that the len be set in the sockaddr struct*/
 250         struct sockaddr_in *addr_in = (struct sockaddr_in *)&w->addr;
 251         addr_in->sin_len = w->addrlen;
 252         sa_endpoints_t endpoints;
 253         endpoints.sae_srcif = 0;
 254         endpoints.sae_srcaddr = NULL;
 255         endpoints.sae_srcaddrlen = 0;
 256         endpoints.sae_dstaddr = (struct sockaddr *)&w->addr;
 257         endpoints.sae_dstaddrlen = w->addrlen;
 258         if (connectx(s, &endpoints, SAE_ASSOCID_ANY,
 259                      CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE,
 260                      NULL, 0, NULL, NULL) == -1) {
 261 #else /* USE_OSX_MSG_FASTOPEN*/
 262 #ifdef USE_MSG_FASTOPEN
 263         pend->c->tcp_do_fastopen = 1;
 264         /* Only do TFO for TCP in which case no connect() is required here.
 265            Don't combine client TFO with SSL, since OpenSSL can't
 266            currently support doing a handshake on fd that already isn't connected*/
 267         if (w->outnet->sslctx && w->ssl_upstream) {
 268                 if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 269 #else /* USE_MSG_FASTOPEN*/
 270         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 271 #endif /* USE_MSG_FASTOPEN*/
 272 #endif /* USE_OSX_MSG_FASTOPEN*/
 273 #ifndef USE_WINSOCK
 274 #ifdef EINPROGRESS
 275                 if(errno != EINPROGRESS) {
 276 #else
 277                 if(1) {
 278 #endif
 279                         if(tcp_connect_errno_needs_log(
 280                                 (struct sockaddr*)&w->addr, w->addrlen))
 281                                 log_err_addr("outgoing tcp: connect",
 282                                         strerror(errno), &w->addr, w->addrlen);
 283                         close(s);
 284 #else /* USE_WINSOCK */
 285                 if(WSAGetLastError() != WSAEINPROGRESS &&
 286                         WSAGetLastError() != WSAEWOULDBLOCK) {
 287                         closesocket(s);
 288 #endif
 289                         return 0;
 290                 }
 291         }
 292 #ifdef USE_MSG_FASTOPEN
 293         }
 294 #endif /* USE_MSG_FASTOPEN */
 295         if(w->outnet->sslctx && w->ssl_upstream) {
 296                 pend->c->ssl = outgoing_ssl_fd(w->outnet->sslctx, s);
 297                 if(!pend->c->ssl) {
 298                         pend->c->fd = s;
 299                         comm_point_close(pend->c);
 300                         return 0;
 301                 }
 302 #ifdef USE_WINSOCK
 303                 comm_point_tcp_win_bio_cb(pend->c, pend->c->ssl);
 304 #endif
 305                 pend->c->ssl_shake_state = comm_ssl_shake_write;
 306         }
 307         w->pkt = NULL;
 308         w->next_waiting = (void*)pend;
 309         pend->id = LDNS_ID_WIRE(pkt);
 310         w->outnet->num_tcp_outgoing++;
 311         w->outnet->tcp_free = pend->next_free;
 312         pend->next_free = NULL;
 313         pend->query = w;
 314         pend->c->repinfo.addrlen = w->addrlen;
 315         memcpy(&pend->c->repinfo.addr, &w->addr, w->addrlen);
 316         sldns_buffer_clear(pend->c->buffer);
 317         sldns_buffer_write(pend->c->buffer, pkt, pkt_len);
 318         sldns_buffer_flip(pend->c->buffer);
 319         pend->c->tcp_is_reading = 0;
 320         pend->c->tcp_byte_count = 0;
 321         comm_point_start_listening(pend->c, s, -1);
 322         return 1;
 323 }
 324
 325 /** see if buffers can be used to service TCP queries */
 326 static void
 327 use_free_buffer(struct outside_network* outnet)
 328 {
 329         struct waiting_tcp* w;
 330         while(outnet->tcp_free && outnet->tcp_wait_first
 331                 && !outnet->want_to_quit) {
 332                 w = outnet->tcp_wait_first;
 333                 outnet->tcp_wait_first = w->next_waiting;
 334                 if(outnet->tcp_wait_last == w)
 335                         outnet->tcp_wait_last = NULL;
 336                 if(!outnet_tcp_take_into_use(w, w->pkt, w->pkt_len)) {
 337                         comm_point_callback_t* cb = w->cb;
 338                         void* cb_arg = w->cb_arg;
 339                         waiting_tcp_delete(w);
 340                         fptr_ok(fptr_whitelist_pending_tcp(cb));
 341                         (void)(*cb)(NULL, cb_arg, NETEVENT_CLOSED, NULL);
 342                 }
 343         }
 344 }
 345
 346 /** decomission a tcp buffer, closes commpoint and frees waiting_tcp entry */
 347 static void
 348 decomission_pending_tcp(struct outside_network* outnet,
 349         struct pending_tcp* pend)
 350 {
 351         if(pend->c->ssl) {
 352 #ifdef HAVE_SSL
 353                 SSL_shutdown(pend->c->ssl);
 354                 SSL_free(pend->c->ssl);
 355                 pend->c->ssl = NULL;
 356 #endif
 357         }
 358         comm_point_close(pend->c);
 359         pend->next_free = outnet->tcp_free;
 360         outnet->tcp_free = pend;
 361         waiting_tcp_delete(pend->query);
 362         pend->query = NULL;
 363         use_free_buffer(outnet);
 364 }
 365
 366 int
 367 outnet_tcp_cb(struct comm_point* c, void* arg, int error,
 368         struct comm_reply *reply_info)
 369 {
 370         struct pending_tcp* pend = (struct pending_tcp*)arg;
 371         struct outside_network* outnet = pend->query->outnet;
 372         verbose(VERB_ALGO, "outnettcp cb");
 373         if(error != NETEVENT_NOERROR) {
 374                 verbose(VERB_QUERY, "outnettcp got tcp error %d", error);
 375                 /* pass error below and exit */
 376         } else {
 377                 /* check ID */
 378                 if(sldns_buffer_limit(c->buffer) < sizeof(uint16_t) ||
 379                         LDNS_ID_WIRE(sldns_buffer_begin(c->buffer))!=pend->id) {
 380                         log_addr(VERB_QUERY,
 381                                 "outnettcp: bad ID in reply, from:",
 382                                 &pend->query->addr, pend->query->addrlen);
 383                         error = NETEVENT_CLOSED;
 384                 }
 385         }
 386         fptr_ok(fptr_whitelist_pending_tcp(pend->query->cb));
 387         (void)(*pend->query->cb)(c, pend->query->cb_arg, error, reply_info);
 388         decomission_pending_tcp(outnet, pend);
 389         return 0;
 390 }
 391
 392 /** lower use count on pc, see if it can be closed */
 393 static void
 394 portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc)
 395 {
 396         struct port_if* pif;
 397         pc->num_outstanding--;
 398         if(pc->num_outstanding > 0) {
 399                 return;
 400         }
 401         /* close it and replace in unused list */
 402         verbose(VERB_ALGO, "close of port %d", pc->number);
 403         comm_point_close(pc->cp);
 404         pif = pc->pif;
 405         log_assert(pif->inuse > 0);
 406         pif->avail_ports[pif->avail_total - pif->inuse] = pc->number;
 407         pif->inuse--;
 408         pif->out[pc->index] = pif->out[pif->inuse];
 409         pif->out[pc->index]->index = pc->index;
 410         pc->next = outnet->unused_fds;
 411         outnet->unused_fds = pc;
 412 }
 413
 414 /** try to send waiting UDP queries */
 415 static void
 416 outnet_send_wait_udp(struct outside_network* outnet)
 417 {
 418         struct pending* pend;
 419         /* process waiting queries */
 420         while(outnet->udp_wait_first && outnet->unused_fds
 421                 && !outnet->want_to_quit) {
 422                 pend = outnet->udp_wait_first;
 423                 outnet->udp_wait_first = pend->next_waiting;
 424                 if(!pend->next_waiting) outnet->udp_wait_last = NULL;
 425                 sldns_buffer_clear(outnet->udp_buff);
 426                 sldns_buffer_write(outnet->udp_buff, pend->pkt, pend->pkt_len);
 427                 sldns_buffer_flip(outnet->udp_buff);
 428                 free(pend->pkt); /* freeing now makes get_mem correct */
 429                 pend->pkt = NULL;
 430                 pend->pkt_len = 0;
 431                 if(!randomize_and_send_udp(pend, outnet->udp_buff,
 432                         pend->timeout)) {
 433                         /* callback error on pending */
 434                         if(pend->cb) {
 435                                 fptr_ok(fptr_whitelist_pending_udp(pend->cb));
 436                                 (void)(*pend->cb)(outnet->unused_fds->cp, pend->cb_arg,
 437                                         NETEVENT_CLOSED, NULL);
 438                         }
 439                         pending_delete(outnet, pend);
 440                 }
 441         }
 442 }
 443
 444 int
 445 outnet_udp_cb(struct comm_point* c, void* arg, int error,
 446         struct comm_reply *reply_info)
 447 {
 448         struct outside_network* outnet = (struct outside_network*)arg;
 449         struct pending key;
 450         struct pending* p;
 451         verbose(VERB_ALGO, "answer cb");
 452
 453         if(error != NETEVENT_NOERROR) {
 454                 verbose(VERB_QUERY, "outnetudp got udp error %d", error);
 455                 return 0;
 456         }
 457         if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
 458                 verbose(VERB_QUERY, "outnetudp udp too short");
 459                 return 0;
 460         }
 461         log_assert(reply_info);
 462
 463         /* setup lookup key */
 464         key.id = (unsigned)LDNS_ID_WIRE(sldns_buffer_begin(c->buffer));
 465         memcpy(&key.addr, &reply_info->addr, reply_info->addrlen);
 466         key.addrlen = reply_info->addrlen;
 467         verbose(VERB_ALGO, "Incoming reply id = %4.4x", key.id);
 468         log_addr(VERB_ALGO, "Incoming reply addr =",
 469                 &reply_info->addr, reply_info->addrlen);
 470
 471         /* find it, see if this thing is a valid query response */
 472         verbose(VERB_ALGO, "lookup size is %d entries", (int)outnet->pending->count);
 473         p = (struct pending*)rbtree_search(outnet->pending, &key);
 474         if(!p) {
 475                 verbose(VERB_QUERY, "received unwanted or unsolicited udp reply dropped.");
 476                 log_buf(VERB_ALGO, "dropped message", c->buffer);
 477                 outnet->unwanted_replies++;
 478                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
 479                         >= outnet->unwanted_threshold) {
 480                         log_warn("unwanted reply total reached threshold (%u)"
 481                                 " you may be under attack."
 482                                 " defensive action: clearing the cache",
 483                                 (unsigned)outnet->unwanted_threshold);
 484                         fptr_ok(fptr_whitelist_alloc_cleanup(
 485                                 outnet->unwanted_action));
 486                         (*outnet->unwanted_action)(outnet->unwanted_param);
 487                         outnet->unwanted_total = 0;
 488                 }
 489                 return 0;
 490         }
 491
 492         verbose(VERB_ALGO, "received udp reply.");
 493         log_buf(VERB_ALGO, "udp message", c->buffer);
 494         if(p->pc->cp != c) {
 495                 verbose(VERB_QUERY, "received reply id,addr on wrong port. "
 496                         "dropped.");
 497                 outnet->unwanted_replies++;
 498                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
 499                         >= outnet->unwanted_threshold) {
 500                         log_warn("unwanted reply total reached threshold (%u)"
 501                                 " you may be under attack."
 502                                 " defensive action: clearing the cache",
 503                                 (unsigned)outnet->unwanted_threshold);
 504                         fptr_ok(fptr_whitelist_alloc_cleanup(
 505                                 outnet->unwanted_action));
 506                         (*outnet->unwanted_action)(outnet->unwanted_param);
 507                         outnet->unwanted_total = 0;
 508                 }
 509                 return 0;
 510         }
 511         comm_timer_disable(p->timer);
 512         verbose(VERB_ALGO, "outnet handle udp reply");
 513         /* delete from tree first in case callback creates a retry */
 514         (void)rbtree_delete(outnet->pending, p->node.key);
 515         if(p->cb) {
 516                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
 517                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_NOERROR, reply_info);
 518         }
 519         portcomm_loweruse(outnet, p->pc);
 520         pending_delete(NULL, p);
 521         outnet_send_wait_udp(outnet);
 522         return 0;
 523 }
 524
 525 /** calculate number of ip4 and ip6 interfaces*/
 526 static void
 527 calc_num46(char** ifs, int num_ifs, int do_ip4, int do_ip6,
 528         int* num_ip4, int* num_ip6)
 529 {
 530         int i;
 531         *num_ip4 = 0;
 532         *num_ip6 = 0;
 533         if(num_ifs <= 0) {
 534                 if(do_ip4)
 535                         *num_ip4 = 1;
 536                 if(do_ip6)
 537                         *num_ip6 = 1;
 538                 return;
 539         }
 540         for(i=0; i<num_ifs; i++)
 541         {
 542                 if(str_is_ip6(ifs[i])) {
 543                         if(do_ip6)
 544                                 (*num_ip6)++;
 545                 } else {
 546                         if(do_ip4)
 547                                 (*num_ip4)++;
 548                 }
 549         }
 550
 551 }
 552
 553 void
 554 pending_udp_timer_delay_cb(void* arg)
 555 {
 556         struct pending* p = (struct pending*)arg;
 557         struct outside_network* outnet = p->outnet;
 558         verbose(VERB_ALGO, "timeout udp with delay");
 559         portcomm_loweruse(outnet, p->pc);
 560         pending_delete(outnet, p);
 561         outnet_send_wait_udp(outnet);
 562 }
 563
 564 void
 565 pending_udp_timer_cb(void *arg)
 566 {
 567         struct pending* p = (struct pending*)arg;
 568         struct outside_network* outnet = p->outnet;
 569         /* it timed out */
 570         verbose(VERB_ALGO, "timeout udp");
 571         if(p->cb) {
 572                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
 573                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_TIMEOUT, NULL);
 574         }
 575         /* if delayclose, keep port open for a longer time.
 576          * But if the udpwaitlist exists, then we are struggling to
 577          * keep up with demand for sockets, so do not wait, but service
 578          * the customer (customer service more important than portICMPs) */
 579         if(outnet->delayclose && !outnet->udp_wait_first) {
 580                 p->cb = NULL;
 581                 p->timer->callback = &pending_udp_timer_delay_cb;
 582                 comm_timer_set(p->timer, &outnet->delay_tv);
 583                 return;
 584         }
 585         portcomm_loweruse(outnet, p->pc);
 586         pending_delete(outnet, p);
 587         outnet_send_wait_udp(outnet);
 588 }
 589
 590 /** create pending_tcp buffers */
 591 static int
 592 create_pending_tcp(struct outside_network* outnet, size_t bufsize)
 593 {
 594         size_t i;
 595         if(outnet->num_tcp == 0)
 596                 return 1; /* no tcp needed, nothing to do */
 597         if(!(outnet->tcp_conns = (struct pending_tcp **)calloc(
 598                         outnet->num_tcp, sizeof(struct pending_tcp*))))
 599                 return 0;
 600         for(i=0; i<outnet->num_tcp; i++) {
 601                 if(!(outnet->tcp_conns[i] = (struct pending_tcp*)calloc(1,
 602                         sizeof(struct pending_tcp))))
 603                         return 0;
 604                 outnet->tcp_conns[i]->next_free = outnet->tcp_free;
 605                 outnet->tcp_free = outnet->tcp_conns[i];
 606                 outnet->tcp_conns[i]->c = comm_point_create_tcp_out(
 607                         outnet->base, bufsize, outnet_tcp_cb,
 608                         outnet->tcp_conns[i]);
 609                 if(!outnet->tcp_conns[i]->c)
 610                         return 0;
 611         }
 612         return 1;
 613 }
 614
 615 /** setup an outgoing interface, ready address */
 616 static int setup_if(struct port_if* pif, const char* addrstr,
 617         int* avail, int numavail, size_t numfd)
 618 {
 619         pif->avail_total = numavail;
 620         pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int));
 621         if(!pif->avail_ports)
 622                 return 0;
 623         if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) &&
 624            !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT,
 625                               &pif->addr, &pif->addrlen, &pif->pfxlen))
 626                 return 0;
 627         pif->maxout = (int)numfd;
 628         pif->inuse = 0;
 629         pif->out = (struct port_comm**)calloc(numfd,
 630                 sizeof(struct port_comm*));
 631         if(!pif->out)
 632                 return 0;
 633         return 1;
 634 }
 635
 636 struct outside_network*
 637 outside_network_create(struct comm_base *base, size_t bufsize,
 638         size_t num_ports, char** ifs, int num_ifs, int do_ip4,
 639         int do_ip6, size_t num_tcp, struct infra_cache* infra,
 640         struct ub_randstate* rnd, int use_caps_for_id, int* availports,
 641         int numavailports, size_t unwanted_threshold, int tcp_mss,
 642         void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
 643         void* sslctx, int delayclose, struct dt_env* dtenv)
 644 {
 645         struct outside_network* outnet = (struct outside_network*)
 646                 calloc(1, sizeof(struct outside_network));
 647         size_t k;
 648         if(!outnet) {
 649                 log_err("malloc failed");
 650                 return NULL;
 651         }
 652         comm_base_timept(base, &outnet->now_secs, &outnet->now_tv);
 653         outnet->base = base;
 654         outnet->num_tcp = num_tcp;
 655         outnet->num_tcp_outgoing = 0;
 656         outnet->infra = infra;
 657         outnet->rnd = rnd;
 658         outnet->sslctx = sslctx;
 659 #ifdef USE_DNSTAP
 660         outnet->dtenv = dtenv;
 661 #else
 662         (void)dtenv;
 663 #endif
 664         outnet->svcd_overhead = 0;
 665         outnet->want_to_quit = 0;
 666         outnet->unwanted_threshold = unwanted_threshold;
 667         outnet->unwanted_action = unwanted_action;
 668         outnet->unwanted_param = unwanted_param;
 669         outnet->use_caps_for_id = use_caps_for_id;
 670         outnet->do_udp = do_udp;
 671         outnet->tcp_mss = tcp_mss;
 672 #ifndef S_SPLINT_S
 673         if(delayclose) {
 674                 outnet->delayclose = 1;
 675                 outnet->delay_tv.tv_sec = delayclose/1000;
 676                 outnet->delay_tv.tv_usec = (delayclose%1000)*1000;
 677         }
 678 #endif
 679         if(numavailports == 0) {
 680                 log_err("no outgoing ports available");
 681                 outside_network_delete(outnet);
 682                 return NULL;
 683         }
 684 #ifndef INET6
 685         do_ip6 = 0;
 686 #endif
 687         calc_num46(ifs, num_ifs, do_ip4, do_ip6,
 688                 &outnet->num_ip4, &outnet->num_ip6);
 689         if(outnet->num_ip4 != 0) {
 690                 if(!(outnet->ip4_ifs = (struct port_if*)calloc(
 691                         (size_t)outnet->num_ip4, sizeof(struct port_if)))) {
 692                         log_err("malloc failed");
 693                         outside_network_delete(outnet);
 694                         return NULL;
 695                 }
 696         }
 697         if(outnet->num_ip6 != 0) {
 698                 if(!(outnet->ip6_ifs = (struct port_if*)calloc(
 699                         (size_t)outnet->num_ip6, sizeof(struct port_if)))) {
 700                         log_err("malloc failed");
 701                         outside_network_delete(outnet);
 702                         return NULL;
 703                 }
 704         }
 705         if(     !(outnet->udp_buff = sldns_buffer_new(bufsize)) ||
 706                 !(outnet->pending = rbtree_create(pending_cmp)) ||
 707                 !(outnet->serviced = rbtree_create(serviced_cmp)) ||
 708                 !create_pending_tcp(outnet, bufsize)) {
 709                 log_err("malloc failed");
 710                 outside_network_delete(outnet);
 711                 return NULL;
 712         }
 713
 714         /* allocate commpoints */
 715         for(k=0; k<num_ports; k++) {
 716                 struct port_comm* pc;
 717                 pc = (struct port_comm*)calloc(1, sizeof(*pc));
 718                 if(!pc) {
 719                         log_err("malloc failed");
 720                         outside_network_delete(outnet);
 721                         return NULL;
 722                 }
 723                 pc->cp = comm_point_create_udp(outnet->base, -1,
 724                         outnet->udp_buff, outnet_udp_cb, outnet);
 725                 if(!pc->cp) {
 726                         log_err("malloc failed");
 727                         free(pc);
 728                         outside_network_delete(outnet);
 729                         return NULL;
 730                 }
 731                 pc->next = outnet->unused_fds;
 732                 outnet->unused_fds = pc;
 733         }
 734
 735         /* allocate interfaces */
 736         if(num_ifs == 0) {
 737                 if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0",
 738                         availports, numavailports, num_ports)) {
 739                         log_err("malloc failed");
 740                         outside_network_delete(outnet);
 741                         return NULL;
 742                 }
 743                 if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::",
 744                         availports, numavailports, num_ports)) {
 745                         log_err("malloc failed");
 746                         outside_network_delete(outnet);
 747                         return NULL;
 748                 }
 749         } else {
 750                 size_t done_4 = 0, done_6 = 0;
 751                 int i;
 752                 for(i=0; i<num_ifs; i++) {
 753                         if(str_is_ip6(ifs[i]) && do_ip6) {
 754                                 if(!setup_if(&outnet->ip6_ifs[done_6], ifs[i],
 755                                         availports, numavailports, num_ports)){
 756                                         log_err("malloc failed");
 757                                         outside_network_delete(outnet);
 758                                         return NULL;
 759                                 }
 760                                 done_6++;
 761                         }
 762                         if(!str_is_ip6(ifs[i]) && do_ip4) {
 763                                 if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i],
 764                                         availports, numavailports, num_ports)){
 765                                         log_err("malloc failed");
 766                                         outside_network_delete(outnet);
 767                                         return NULL;
 768                                 }
 769                                 done_4++;
 770                         }
 771                 }
 772         }
 773         return outnet;
 774 }
 775
 776 /** helper pending delete */
 777 static void
 778 pending_node_del(rbnode_t* node, void* arg)
 779 {
 780         struct pending* pend = (struct pending*)node;
 781         struct outside_network* outnet = (struct outside_network*)arg;
 782         pending_delete(outnet, pend);
 783 }
 784
 785 /** helper serviced delete */
 786 static void
 787 serviced_node_del(rbnode_t* node, void* ATTR_UNUSED(arg))
 788 {
 789         struct serviced_query* sq = (struct serviced_query*)node;
 790         struct service_callback* p = sq->cblist, *np;
 791         free(sq->qbuf);
 792         free(sq->zone);
 793         edns_opt_list_free(sq->opt_list);
 794         while(p) {
 795                 np = p->next;
 796                 free(p);
 797                 p = np;
 798         }
 799         free(sq);
 800 }
 801
 802 void
 803 outside_network_quit_prepare(struct outside_network* outnet)
 804 {
 805         if(!outnet)
 806                 return;
 807         /* prevent queued items from being sent */
 808         outnet->want_to_quit = 1;
 809 }
 810
 811 void
 812 outside_network_delete(struct outside_network* outnet)
 813 {
 814         if(!outnet)
 815                 return;
 816         outnet->want_to_quit = 1;
 817         /* check every element, since we can be called on malloc error */
 818         if(outnet->pending) {
 819                 /* free pending elements, but do no unlink from tree. */
 820                 traverse_postorder(outnet->pending, pending_node_del, NULL);
 821                 free(outnet->pending);
 822         }
 823         if(outnet->serviced) {
 824                 traverse_postorder(outnet->serviced, serviced_node_del, NULL);
 825                 free(outnet->serviced);
 826         }
 827         if(outnet->udp_buff)
 828                 sldns_buffer_free(outnet->udp_buff);
 829         if(outnet->unused_fds) {
 830                 struct port_comm* p = outnet->unused_fds, *np;
 831                 while(p) {
 832                         np = p->next;
 833                         comm_point_delete(p->cp);
 834                         free(p);
 835                         p = np;
 836                 }
 837                 outnet->unused_fds = NULL;
 838         }
 839         if(outnet->ip4_ifs) {
 840                 int i, k;
 841                 for(i=0; i<outnet->num_ip4; i++) {
 842                         for(k=0; k<outnet->ip4_ifs[i].inuse; k++) {
 843                                 struct port_comm* pc = outnet->ip4_ifs[i].
 844                                         out[k];
 845                                 comm_point_delete(pc->cp);
 846                                 free(pc);
 847                         }
 848                         free(outnet->ip4_ifs[i].avail_ports);
 849                         free(outnet->ip4_ifs[i].out);
 850                 }
 851                 free(outnet->ip4_ifs);
 852         }
 853         if(outnet->ip6_ifs) {
 854                 int i, k;
 855                 for(i=0; i<outnet->num_ip6; i++) {
 856                         for(k=0; k<outnet->ip6_ifs[i].inuse; k++) {
 857                                 struct port_comm* pc = outnet->ip6_ifs[i].
 858                                         out[k];
 859                                 comm_point_delete(pc->cp);
 860                                 free(pc);
 861                         }
 862                         free(outnet->ip6_ifs[i].avail_ports);
 863                         free(outnet->ip6_ifs[i].out);
 864                 }
 865                 free(outnet->ip6_ifs);
 866         }
 867         if(outnet->tcp_conns) {
 868                 size_t i;
 869                 for(i=0; i<outnet->num_tcp; i++)
 870                         if(outnet->tcp_conns[i]) {
 871                                 comm_point_delete(outnet->tcp_conns[i]->c);
 872                                 waiting_tcp_delete(outnet->tcp_conns[i]->query);
 873                                 free(outnet->tcp_conns[i]);
 874                         }
 875                 free(outnet->tcp_conns);
 876         }
 877         if(outnet->tcp_wait_first) {
 878                 struct waiting_tcp* p = outnet->tcp_wait_first, *np;
 879                 while(p) {
 880                         np = p->next_waiting;
 881                         waiting_tcp_delete(p);
 882                         p = np;
 883                 }
 884         }
 885         if(outnet->udp_wait_first) {
 886                 struct pending* p = outnet->udp_wait_first, *np;
 887                 while(p) {
 888                         np = p->next_waiting;
 889                         pending_delete(NULL, p);
 890                         p = np;
 891                 }
 892         }
 893         free(outnet);
 894 }
 895
 896 void
 897 pending_delete(struct outside_network* outnet, struct pending* p)
 898 {
 899         if(!p)
 900                 return;
 901         if(outnet && outnet->udp_wait_first &&
 902                 (p->next_waiting || p == outnet->udp_wait_last) ) {
 903                 /* delete from waiting list, if it is in the waiting list */
 904                 struct pending* prev = NULL, *x = outnet->udp_wait_first;
 905                 while(x && x != p) {
 906                         prev = x;
 907                         x = x->next_waiting;
 908                 }
 909                 if(x) {
 910                         log_assert(x == p);
 911                         if(prev)
 912                                 prev->next_waiting = p->next_waiting;
 913                         else    outnet->udp_wait_first = p->next_waiting;
 914                         if(outnet->udp_wait_last == p)
 915                                 outnet->udp_wait_last = prev;
 916                 }
 917         }
 918         if(outnet) {
 919                 (void)rbtree_delete(outnet->pending, p->node.key);
 920         }
 921         if(p->timer)
 922                 comm_timer_delete(p->timer);
 923         free(p->pkt);
 924         free(p);
 925 }
 926
 927 static void
 928 sai6_putrandom(struct sockaddr_in6 *sa, int pfxlen, struct ub_randstate *rnd)
 929 {
 930         int i, last;
 931         if(!(pfxlen > 0 && pfxlen < 128))
 932                 return;
 933         for(i = 0; i < (128 - pfxlen) / 8; i++) {
 934                 sa->sin6_addr.s6_addr[15-i] = (uint8_t)ub_random_max(rnd, 256);
 935         }
 936         last = pfxlen & 7;
 937         if(last != 0) {
 938                 sa->sin6_addr.s6_addr[15-i] |=
 939                         ((0xFF >> last) & ub_random_max(rnd, 256));
 940         }
 941 }
 942
 943 /**
 944  * Try to open a UDP socket for outgoing communication.
 945  * Sets sockets options as needed.
 946  * @param addr: socket address.
 947  * @param addrlen: length of address.
 948  * @param pfxlen: length of network prefix (for address randomisation).
 949  * @param port: port override for addr.
 950  * @param inuse: if -1 is returned, this bool means the port was in use.
 951  * @param rnd: random state (for address randomisation).
 952  * @return fd or -1
 953  */
 954 static int
 955 udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen,
 956         int port, int* inuse, struct ub_randstate* rnd)
 957 {
 958         int fd, noproto;
 959         if(addr_is_ip6(addr, addrlen)) {
 960                 int freebind = 0;
 961                 struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
 962                 sa.sin6_port = (in_port_t)htons((uint16_t)port);
 963                 if(pfxlen != 0) {
 964                         freebind = 1;
 965                         sai6_putrandom(&sa, pfxlen, rnd);
 966                 }
 967                 fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
 968                         (struct sockaddr*)&sa, addrlen, 1, inuse, &noproto,
 969                         0, 0, 0, NULL, 0, freebind);
 970         } else {
 971                 struct sockaddr_in* sa = (struct sockaddr_in*)addr;
 972                 sa->sin_port = (in_port_t)htons((uint16_t)port);
 973                 fd = create_udp_sock(AF_INET, SOCK_DGRAM,
 974                         (struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
 975                         0, 0, 0, NULL, 0, 0);
 976         }
 977         return fd;
 978 }
 979
 980 /** Select random ID */
 981 static int
 982 select_id(struct outside_network* outnet, struct pending* pend,
 983         sldns_buffer* packet)
 984 {
 985         int id_tries = 0;
 986         pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
 987         LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
 988
 989         /* insert in tree */
 990         pend->node.key = pend;
 991         while(!rbtree_insert(outnet->pending, &pend->node)) {
 992                 /* change ID to avoid collision */
 993                 pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
 994                 LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
 995                 id_tries++;
 996                 if(id_tries == MAX_ID_RETRY) {
 997                         pend->id=99999; /* non existant ID */
 998                         log_err("failed to generate unique ID, drop msg");
 999                         return 0;
1000                 }
1001         }
1002         verbose(VERB_ALGO, "inserted new pending reply id=%4.4x", pend->id);
1003         return 1;
1004 }
1005
1006 /** Select random interface and port */
1007 static int
1008 select_ifport(struct outside_network* outnet, struct pending* pend,
1009         int num_if, struct port_if* ifs)
1010 {
1011         int my_if, my_port, fd, portno, inuse, tries=0;
1012         struct port_if* pif;
1013         /* randomly select interface and port */
1014         if(num_if == 0) {
1015                 verbose(VERB_QUERY, "Need to send query but have no "
1016                         "outgoing interfaces of that family");
1017                 return 0;
1018         }
1019         log_assert(outnet->unused_fds);
1020         tries = 0;
1021         while(1) {
1022                 my_if = ub_random_max(outnet->rnd, num_if);
1023                 pif = &ifs[my_if];
1024                 my_port = ub_random_max(outnet->rnd, pif->avail_total);
1025                 if(my_port < pif->inuse) {
1026                         /* port already open */
1027                         pend->pc = pif->out[my_port];
1028                         verbose(VERB_ALGO, "using UDP if=%d port=%d",
1029                                 my_if, pend->pc->number);
1030                         break;
1031                 }
1032                 /* try to open new port, if fails, loop to try again */
1033                 log_assert(pif->inuse < pif->maxout);
1034                 portno = pif->avail_ports[my_port - pif->inuse];
1035                 fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen,
1036                         portno, &inuse, outnet->rnd);
1037                 if(fd == -1 && !inuse) {
1038                         /* nonrecoverable error making socket */
1039                         return 0;
1040                 }
1041                 if(fd != -1) {
1042                         verbose(VERB_ALGO, "opened UDP if=%d port=%d",
1043                                 my_if, portno);
1044                         /* grab fd */
1045                         pend->pc = outnet->unused_fds;
1046                         outnet->unused_fds = pend->pc->next;
1047
1048                         /* setup portcomm */
1049                         pend->pc->next = NULL;
1050                         pend->pc->number = portno;
1051                         pend->pc->pif = pif;
1052                         pend->pc->index = pif->inuse;
1053                         pend->pc->num_outstanding = 0;
1054                         comm_point_start_listening(pend->pc->cp, fd, -1);
1055
1056                         /* grab port in interface */
1057                         pif->out[pif->inuse] = pend->pc;
1058                         pif->avail_ports[my_port - pif->inuse] =
1059                                 pif->avail_ports[pif->avail_total-pif->inuse-1];
1060                         pif->inuse++;
1061                         break;
1062                 }
1063                 /* failed, already in use */
1064                 verbose(VERB_QUERY, "port %d in use, trying another", portno);
1065                 tries++;
1066                 if(tries == MAX_PORT_RETRY) {
1067                         log_err("failed to find an open port, drop msg");
1068                         return 0;
1069                 }
1070         }
1071         log_assert(pend->pc);
1072         pend->pc->num_outstanding++;
1073
1074         return 1;
1075 }
1076
1077 static int
1078 randomize_and_send_udp(struct pending* pend, sldns_buffer* packet, int timeout)
1079 {
1080         struct timeval tv;
1081         struct outside_network* outnet = pend->sq->outnet;
1082
1083         /* select id */
1084         if(!select_id(outnet, pend, packet)) {
1085                 return 0;
1086         }
1087
1088         /* select src_if, port */
1089         if(addr_is_ip6(&pend->addr, pend->addrlen)) {
1090                 if(!select_ifport(outnet, pend,
1091                         outnet->num_ip6, outnet->ip6_ifs))
1092                         return 0;
1093         } else {
1094                 if(!select_ifport(outnet, pend,
1095                         outnet->num_ip4, outnet->ip4_ifs))
1096                         return 0;
1097         }
1098         log_assert(pend->pc && pend->pc->cp);
1099
1100         /* send it over the commlink */
1101         if(!comm_point_send_udp_msg(pend->pc->cp, packet,
1102                 (struct sockaddr*)&pend->addr, pend->addrlen)) {
1103                 portcomm_loweruse(outnet, pend->pc);
1104                 return 0;
1105         }
1106
1107         /* system calls to set timeout after sending UDP to make roundtrip
1108            smaller. */
1109 #ifndef S_SPLINT_S
1110         tv.tv_sec = timeout/1000;
1111         tv.tv_usec = (timeout%1000)*1000;
1112 #endif
1113         comm_timer_set(pend->timer, &tv);
1114
1115 #ifdef USE_DNSTAP
1116         if(outnet->dtenv &&
1117            (outnet->dtenv->log_resolver_query_messages ||
1118             outnet->dtenv->log_forwarder_query_messages))
1119                 dt_msg_send_outside_query(outnet->dtenv, &pend->addr, comm_udp,
1120                 pend->sq->zone, pend->sq->zonelen, packet);
1121 #endif
1122         return 1;
1123 }
1124
1125 struct pending*
1126 pending_udp_query(struct serviced_query* sq, struct sldns_buffer* packet,
1127         int timeout, comm_point_callback_t* cb, void* cb_arg)
1128 {
1129         struct pending* pend = (struct pending*)calloc(1, sizeof(*pend));
1130         if(!pend) return NULL;
1131         pend->outnet = sq->outnet;
1132         pend->sq = sq;
1133         pend->addrlen = sq->addrlen;
1134         memmove(&pend->addr, &sq->addr, sq->addrlen);
1135         pend->cb = cb;
1136         pend->cb_arg = cb_arg;
1137         pend->node.key = pend;
1138         pend->timer = comm_timer_create(sq->outnet->base, pending_udp_timer_cb,
1139                 pend);
1140         if(!pend->timer) {
1141                 free(pend);
1142                 return NULL;
1143         }
1144
1145         if(sq->outnet->unused_fds == NULL) {
1146                 /* no unused fd, cannot create a new port (randomly) */
1147                 verbose(VERB_ALGO, "no fds available, udp query waiting");
1148                 pend->timeout = timeout;
1149                 pend->pkt_len = sldns_buffer_limit(packet);
1150                 pend->pkt = (uint8_t*)memdup(sldns_buffer_begin(packet),
1151                         pend->pkt_len);
1152                 if(!pend->pkt) {
1153                         comm_timer_delete(pend->timer);
1154                         free(pend);
1155                         return NULL;
1156                 }
1157                 /* put at end of waiting list */
1158                 if(sq->outnet->udp_wait_last)
1159                         sq->outnet->udp_wait_last->next_waiting = pend;
1160                 else
1161                         sq->outnet->udp_wait_first = pend;
1162                 sq->outnet->udp_wait_last = pend;
1163                 return pend;
1164         }
1165         if(!randomize_and_send_udp(pend, packet, timeout)) {
1166                 pending_delete(sq->outnet, pend);
1167                 return NULL;
1168         }
1169         return pend;
1170 }
1171
1172 void
1173 outnet_tcptimer(void* arg)
1174 {
1175         struct waiting_tcp* w = (struct waiting_tcp*)arg;
1176         struct outside_network* outnet = w->outnet;
1177         comm_point_callback_t* cb;
1178         void* cb_arg;
1179         if(w->pkt) {
1180                 /* it is on the waiting list */
1181                 waiting_list_remove(outnet, w);
1182         } else {
1183                 /* it was in use */
1184                 struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
1185                 comm_point_close(pend->c);
1186                 pend->query = NULL;
1187                 pend->next_free = outnet->tcp_free;
1188                 outnet->tcp_free = pend;
1189         }
1190         cb = w->cb;
1191         cb_arg = w->cb_arg;
1192         waiting_tcp_delete(w);
1193         fptr_ok(fptr_whitelist_pending_tcp(cb));
1194         (void)(*cb)(NULL, cb_arg, NETEVENT_TIMEOUT, NULL);
1195         use_free_buffer(outnet);
1196 }
1197
1198 struct waiting_tcp*
1199 pending_tcp_query(struct serviced_query* sq, sldns_buffer* packet,
1200         int timeout, comm_point_callback_t* callback, void* callback_arg)
1201 {
1202         struct pending_tcp* pend = sq->outnet->tcp_free;
1203         struct waiting_tcp* w;
1204         struct timeval tv;
1205         uint16_t id;
1206         /* if no buffer is free allocate space to store query */
1207         w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp)
1208                 + (pend?0:sldns_buffer_limit(packet)));
1209         if(!w) {
1210                 return NULL;
1211         }
1212         if(!(w->timer = comm_timer_create(sq->outnet->base, outnet_tcptimer, w))) {
1213                 free(w);
1214                 return NULL;
1215         }
1216         w->pkt = NULL;
1217         w->pkt_len = 0;
1218         id = ((unsigned)ub_random(sq->outnet->rnd)>>8) & 0xffff;
1219         LDNS_ID_SET(sldns_buffer_begin(packet), id);
1220         memcpy(&w->addr, &sq->addr, sq->addrlen);
1221         w->addrlen = sq->addrlen;
1222         w->outnet = sq->outnet;
1223         w->cb = callback;
1224         w->cb_arg = callback_arg;
1225         w->ssl_upstream = sq->ssl_upstream;
1226 #ifndef S_SPLINT_S
1227         tv.tv_sec = timeout;
1228         tv.tv_usec = 0;
1229 #endif
1230         comm_timer_set(w->timer, &tv);
1231         if(pend) {
1232                 /* we have a buffer available right now */
1233                 if(!outnet_tcp_take_into_use(w, sldns_buffer_begin(packet),
1234                         sldns_buffer_limit(packet))) {
1235                         waiting_tcp_delete(w);
1236                         return NULL;
1237                 }
1238 #ifdef USE_DNSTAP
1239                 if(sq->outnet->dtenv &&
1240                    (sq->outnet->dtenv->log_resolver_query_messages ||
1241                     sq->outnet->dtenv->log_forwarder_query_messages))
1242                 dt_msg_send_outside_query(sq->outnet->dtenv, &sq->addr,
1243                 comm_tcp, sq->zone, sq->zonelen, packet);
1244 #endif
1245         } else {
1246                 /* queue up */
1247                 w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
1248                 w->pkt_len = sldns_buffer_limit(packet);
1249                 memmove(w->pkt, sldns_buffer_begin(packet), w->pkt_len);
1250                 w->next_waiting = NULL;
1251                 if(sq->outnet->tcp_wait_last)
1252                         sq->outnet->tcp_wait_last->next_waiting = w;
1253                 else    sq->outnet->tcp_wait_first = w;
1254                 sq->outnet->tcp_wait_last = w;
1255         }
1256         return w;
1257 }
1258
1259 /** create query for serviced queries */
1260 static void
1261 serviced_gen_query(sldns_buffer* buff, uint8_t* qname, size_t qnamelen,
1262         uint16_t qtype, uint16_t qclass, uint16_t flags)
1263 {
1264         sldns_buffer_clear(buff);
1265         /* skip id */
1266         sldns_buffer_write_u16(buff, flags);
1267         sldns_buffer_write_u16(buff, 1); /* qdcount */
1268         sldns_buffer_write_u16(buff, 0); /* ancount */
1269         sldns_buffer_write_u16(buff, 0); /* nscount */
1270         sldns_buffer_write_u16(buff, 0); /* arcount */
1271         sldns_buffer_write(buff, qname, qnamelen);
1272         sldns_buffer_write_u16(buff, qtype);
1273         sldns_buffer_write_u16(buff, qclass);
1274         sldns_buffer_flip(buff);
1275 }
1276
1277 /** lookup serviced query in serviced query rbtree */
1278 static struct serviced_query*
1279 lookup_serviced(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1280         struct sockaddr_storage* addr, socklen_t addrlen,
1281         struct edns_option* opt_list)
1282 {
1283         struct serviced_query key;
1284         key.node.key = &key;
1285         key.qbuf = sldns_buffer_begin(buff);
1286         key.qbuflen = sldns_buffer_limit(buff);
1287         key.dnssec = dnssec;
1288         memcpy(&key.addr, addr, addrlen);
1289         key.addrlen = addrlen;
1290         key.outnet = outnet;
1291         key.opt_list = opt_list;
1292         return (struct serviced_query*)rbtree_search(outnet->serviced, &key);
1293 }
1294
1295 /** Create new serviced entry */
1296 static struct serviced_query*
1297 serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1298         int want_dnssec, int nocaps, int tcp_upstream, int ssl_upstream,
1299         struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
1300         size_t zonelen, int qtype, struct edns_option* opt_list)
1301 {
1302         struct serviced_query* sq = (struct serviced_query*)malloc(sizeof(*sq));
1303 #ifdef UNBOUND_DEBUG
1304         rbnode_t* ins;
1305 #endif
1306         if(!sq)
1307                 return NULL;
1308         sq->node.key = sq;
1309         sq->qbuf = memdup(sldns_buffer_begin(buff), sldns_buffer_limit(buff));
1310         if(!sq->qbuf) {
1311                 free(sq);
1312                 return NULL;
1313         }
1314         sq->qbuflen = sldns_buffer_limit(buff);
1315         sq->zone = memdup(zone, zonelen);
1316         if(!sq->zone) {
1317                 free(sq->qbuf);
1318                 free(sq);
1319                 return NULL;
1320         }
1321         sq->zonelen = zonelen;
1322         sq->qtype = qtype;
1323         sq->dnssec = dnssec;
1324         sq->want_dnssec = want_dnssec;
1325         sq->nocaps = nocaps;
1326         sq->tcp_upstream = tcp_upstream;
1327         sq->ssl_upstream = ssl_upstream;
1328         memcpy(&sq->addr, addr, addrlen);
1329         sq->addrlen = addrlen;
1330         sq->opt_list = NULL;
1331         if(opt_list) {
1332                 sq->opt_list = edns_opt_copy_alloc(opt_list);
1333                 if(!sq->opt_list) {
1334                         free(sq->zone);
1335                         free(sq->qbuf);
1336                         free(sq);
1337                         return NULL;
1338                 }
1339         }
1340         sq->outnet = outnet;
1341         sq->cblist = NULL;
1342         sq->pending = NULL;
1343         sq->status = serviced_initial;
1344         sq->retry = 0;
1345         sq->to_be_deleted = 0;
1346 #ifdef UNBOUND_DEBUG
1347         ins =
1348 #else
1349         (void)
1350 #endif
1351         rbtree_insert(outnet->serviced, &sq->node);
1352         log_assert(ins != NULL); /* must not be already present */
1353         return sq;
1354 }
1355
1356 /** remove waiting tcp from the outnet waiting list */
1357 static void
1358 waiting_list_remove(struct outside_network* outnet, struct waiting_tcp* w)
1359 {
1360         struct waiting_tcp* p = outnet->tcp_wait_first, *prev = NULL;
1361         while(p) {
1362                 if(p == w) {
1363                         /* remove w */
1364                         if(prev)
1365                                 prev->next_waiting = w->next_waiting;
1366                         else    outnet->tcp_wait_first = w->next_waiting;
1367                         if(outnet->tcp_wait_last == w)
1368                                 outnet->tcp_wait_last = prev;
1369                         return;
1370                 }
1371                 prev = p;
1372                 p = p->next_waiting;
1373         }
1374 }
1375
1376 /** cleanup serviced query entry */
1377 static void
1378 serviced_delete(struct serviced_query* sq)
1379 {
1380         if(sq->pending) {
1381                 /* clear up the pending query */
1382                 if(sq->status == serviced_query_UDP_EDNS ||
1383                         sq->status == serviced_query_UDP ||
1384                         sq->status == serviced_query_PROBE_EDNS ||
1385                         sq->status == serviced_query_UDP_EDNS_FRAG ||
1386                         sq->status == serviced_query_UDP_EDNS_fallback) {
1387                         struct pending* p = (struct pending*)sq->pending;
1388                         if(p->pc)
1389                                 portcomm_loweruse(sq->outnet, p->pc);
1390                         pending_delete(sq->outnet, p);
1391                         /* this call can cause reentrant calls back into the
1392                          * mesh */
1393                         outnet_send_wait_udp(sq->outnet);
1394                 } else {
1395                         struct waiting_tcp* p = (struct waiting_tcp*)
1396                                 sq->pending;
1397                         if(p->pkt == NULL) {
1398                                 decomission_pending_tcp(sq->outnet,
1399                                         (struct pending_tcp*)p->next_waiting);
1400                         } else {
1401                                 waiting_list_remove(sq->outnet, p);
1402                                 waiting_tcp_delete(p);
1403                         }
1404                 }
1405         }
1406         /* does not delete from tree, caller has to do that */
1407         serviced_node_del(&sq->node, NULL);
1408 }
1409
1410 /** perturb a dname capitalization randomly */
1411 static void
1412 serviced_perturb_qname(struct ub_randstate* rnd, uint8_t* qbuf, size_t len)
1413 {
1414         uint8_t lablen;
1415         uint8_t* d = qbuf + 10;
1416         long int random = 0;
1417         int bits = 0;
1418         log_assert(len >= 10 + 5 /* offset qname, root, qtype, qclass */);
1419         (void)len;
1420         lablen = *d++;
1421         while(lablen) {
1422                 while(lablen--) {
1423                         /* only perturb A-Z, a-z */
1424                         if(isalpha((unsigned char)*d)) {
1425                                 /* get a random bit */
1426                                 if(bits == 0) {
1427                                         random = ub_random(rnd);
1428                                         bits = 30;
1429                                 }
1430                                 if(random & 0x1) {
1431                                         *d = (uint8_t)toupper((unsigned char)*d);
1432                                 } else {
1433                                         *d = (uint8_t)tolower((unsigned char)*d);
1434                                 }
1435                                 random >>= 1;
1436                                 bits--;
1437                         }
1438                         d++;
1439                 }
1440                 lablen = *d++;
1441         }
1442         if(verbosity >= VERB_ALGO) {
1443                 char buf[LDNS_MAX_DOMAINLEN+1];
1444                 dname_str(qbuf+10, buf);
1445                 verbose(VERB_ALGO, "qname perturbed to %s", buf);
1446         }
1447 }
1448
1449 /** put serviced query into a buffer */
1450 static void
1451 serviced_encode(struct serviced_query* sq, sldns_buffer* buff, int with_edns)
1452 {
1453         /* if we are using 0x20 bits for ID randomness, perturb them */
1454         if(sq->outnet->use_caps_for_id && !sq->nocaps) {
1455                 serviced_perturb_qname(sq->outnet->rnd, sq->qbuf, sq->qbuflen);
1456         }
1457         /* generate query */
1458         sldns_buffer_clear(buff);
1459         sldns_buffer_write_u16(buff, 0); /* id placeholder */
1460         sldns_buffer_write(buff, sq->qbuf, sq->qbuflen);
1461         sldns_buffer_flip(buff);
1462         if(with_edns) {
1463                 /* add edns section */
1464                 struct edns_data edns;
1465                 edns.edns_present = 1;
1466                 edns.ext_rcode = 0;
1467                 edns.edns_version = EDNS_ADVERTISED_VERSION;
1468                 edns.opt_list = sq->opt_list;
1469                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1470                         if(addr_is_ip6(&sq->addr, sq->addrlen)) {
1471                                 if(EDNS_FRAG_SIZE_IP6 < EDNS_ADVERTISED_SIZE)
1472                                         edns.udp_size = EDNS_FRAG_SIZE_IP6;
1473                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
1474                         } else {
1475                                 if(EDNS_FRAG_SIZE_IP4 < EDNS_ADVERTISED_SIZE)
1476                                         edns.udp_size = EDNS_FRAG_SIZE_IP4;
1477                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
1478                         }
1479                 } else {
1480                         edns.udp_size = EDNS_ADVERTISED_SIZE;
1481                 }
1482                 edns.bits = 0;
1483                 if(sq->dnssec & EDNS_DO)
1484                         edns.bits = EDNS_DO;
1485                 if(sq->dnssec & BIT_CD)
1486                         LDNS_CD_SET(sldns_buffer_begin(buff));
1487                 attach_edns_record(buff, &edns);
1488         }
1489 }
1490
1491 /**
1492  * Perform serviced query UDP sending operation.
1493  * Sends UDP with EDNS, unless infra host marked non EDNS.
1494  * @param sq: query to send.
1495  * @param buff: buffer scratch space.
1496  * @return 0 on error.
1497  */
1498 static int
1499 serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
1500 {
1501         int rtt, vs;
1502         uint8_t edns_lame_known;
1503         time_t now = *sq->outnet->now_secs;
1504
1505         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1506                 sq->zonelen, now, &vs, &edns_lame_known, &rtt))
1507                 return 0;
1508         sq->last_rtt = rtt;
1509         verbose(VERB_ALGO, "EDNS lookup known=%d vs=%d", edns_lame_known, vs);
1510         if(sq->status == serviced_initial) {
1511                 if(edns_lame_known == 0 && rtt > 5000 && rtt < 10001) {
1512                         /* perform EDNS lame probe - check if server is
1513                          * EDNS lame (EDNS queries to it are dropped) */
1514                         verbose(VERB_ALGO, "serviced query: send probe to see "
1515                                 " if use of EDNS causes timeouts");
1516                         /* even 700 msec may be too small */
1517                         rtt = 1000;
1518                         sq->status = serviced_query_PROBE_EDNS;
1519                 } else if(vs != -1) {
1520                         sq->status = serviced_query_UDP_EDNS;
1521                 } else {
1522                         sq->status = serviced_query_UDP;
1523                 }
1524         }
1525         serviced_encode(sq, buff, (sq->status == serviced_query_UDP_EDNS) ||
1526                 (sq->status == serviced_query_UDP_EDNS_FRAG));
1527         sq->last_sent_time = *sq->outnet->now_tv;
1528         sq->edns_lame_known = (int)edns_lame_known;
1529         verbose(VERB_ALGO, "serviced query UDP timeout=%d msec", rtt);
1530         sq->pending = pending_udp_query(sq, buff, rtt,
1531                 serviced_udp_callback, sq);
1532         if(!sq->pending)
1533                 return 0;
1534         return 1;
1535 }
1536
1537 /** check that perturbed qname is identical */
1538 static int
1539 serviced_check_qname(sldns_buffer* pkt, uint8_t* qbuf, size_t qbuflen)
1540 {
1541         uint8_t* d1 = sldns_buffer_at(pkt, 12);
1542         uint8_t* d2 = qbuf+10;
1543         uint8_t len1, len2;
1544         int count = 0;
1545         log_assert(qbuflen >= 15 /* 10 header, root, type, class */);
1546         len1 = *d1++;
1547         len2 = *d2++;
1548         if(sldns_buffer_limit(pkt) < 12+1+4) /* packet too small for qname */
1549                 return 0;
1550         while(len1 != 0 || len2 != 0) {
1551                 if(LABEL_IS_PTR(len1)) {
1552                         d1 = sldns_buffer_at(pkt, PTR_OFFSET(len1, *d1));
1553                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1554                                 return 0;
1555                         len1 = *d1++;
1556                         if(count++ > MAX_COMPRESS_PTRS)
1557                                 return 0;
1558                         continue;
1559                 }
1560                 if(d2 > qbuf+qbuflen)
1561                         return 0;
1562                 if(len1 != len2)
1563                         return 0;
1564                 if(len1 > LDNS_MAX_LABELLEN)
1565                         return 0;
1566                 log_assert(len1 <= LDNS_MAX_LABELLEN);
1567                 log_assert(len2 <= LDNS_MAX_LABELLEN);
1568                 log_assert(len1 == len2 && len1 != 0);
1569                 /* compare the labels - bitwise identical */
1570                 if(memcmp(d1, d2, len1) != 0)
1571                         return 0;
1572                 d1 += len1;
1573                 d2 += len2;
1574                 len1 = *d1++;
1575                 len2 = *d2++;
1576         }
1577         return 1;
1578 }
1579
1580 /** call the callbacks for a serviced query */
1581 static void
1582 serviced_callbacks(struct serviced_query* sq, int error, struct comm_point* c,
1583         struct comm_reply* rep)
1584 {
1585         struct service_callback* p;
1586         int dobackup = (sq->cblist && sq->cblist->next); /* >1 cb*/
1587         uint8_t *backup_p = NULL;
1588         size_t backlen = 0;
1589 #ifdef UNBOUND_DEBUG
1590         rbnode_t* rem =
1591 #else
1592         (void)
1593 #endif
1594         /* remove from tree, and schedule for deletion, so that callbacks
1595          * can safely deregister themselves and even create new serviced
1596          * queries that are identical to this one. */
1597         rbtree_delete(sq->outnet->serviced, sq);
1598         log_assert(rem); /* should have been present */
1599         sq->to_be_deleted = 1;
1600         verbose(VERB_ALGO, "svcd callbacks start");
1601         if(sq->outnet->use_caps_for_id && error == NETEVENT_NOERROR && c &&
1602                 !sq->nocaps && sq->qtype != LDNS_RR_TYPE_PTR) {
1603                 /* for type PTR do not check perturbed name in answer,
1604                  * compatibility with cisco dns guard boxes that mess up
1605                  * reverse queries 0x20 contents */
1606                 /* noerror and nxdomain must have a qname in reply */
1607                 if(sldns_buffer_read_u16_at(c->buffer, 4) == 0 &&
1608                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1609                                 == LDNS_RCODE_NOERROR ||
1610                          LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1611                                 == LDNS_RCODE_NXDOMAIN)) {
1612                         verbose(VERB_DETAIL, "no qname in reply to check 0x20ID");
1613                         log_addr(VERB_DETAIL, "from server",
1614                                 &sq->addr, sq->addrlen);
1615                         log_buf(VERB_DETAIL, "for packet", c->buffer);
1616                         error = NETEVENT_CLOSED;
1617                         c = NULL;
1618                 } else if(sldns_buffer_read_u16_at(c->buffer, 4) > 0 &&
1619                         !serviced_check_qname(c->buffer, sq->qbuf,
1620                         sq->qbuflen)) {
1621                         verbose(VERB_DETAIL, "wrong 0x20-ID in reply qname");
1622                         log_addr(VERB_DETAIL, "from server",
1623                                 &sq->addr, sq->addrlen);
1624                         log_buf(VERB_DETAIL, "for packet", c->buffer);
1625                         error = NETEVENT_CAPSFAIL;
1626                         /* and cleanup too */
1627                         pkt_dname_tolower(c->buffer,
1628                                 sldns_buffer_at(c->buffer, 12));
1629                 } else {
1630                         verbose(VERB_ALGO, "good 0x20-ID in reply qname");
1631                         /* cleanup caps, prettier cache contents. */
1632                         pkt_dname_tolower(c->buffer,
1633                                 sldns_buffer_at(c->buffer, 12));
1634                 }
1635         }
1636         if(dobackup && c) {
1637                 /* make a backup of the query, since the querystate processing
1638                  * may send outgoing queries that overwrite the buffer.
1639                  * use secondary buffer to store the query.
1640                  * This is a data copy, but faster than packet to server */
1641                 backlen = sldns_buffer_limit(c->buffer);
1642                 backup_p = memdup(sldns_buffer_begin(c->buffer), backlen);
1643                 if(!backup_p) {
1644                         log_err("malloc failure in serviced query callbacks");
1645                         error = NETEVENT_CLOSED;
1646                         c = NULL;
1647                 }
1648                 sq->outnet->svcd_overhead = backlen;
1649         }
1650         /* test the actual sq->cblist, because the next elem could be deleted*/
1651         while((p=sq->cblist) != NULL) {
1652                 sq->cblist = p->next; /* remove this element */
1653                 if(dobackup && c) {
1654                         sldns_buffer_clear(c->buffer);
1655                         sldns_buffer_write(c->buffer, backup_p, backlen);
1656                         sldns_buffer_flip(c->buffer);
1657                 }
1658                 fptr_ok(fptr_whitelist_serviced_query(p->cb));
1659                 (void)(*p->cb)(c, p->cb_arg, error, rep);
1660                 free(p);
1661         }
1662         if(backup_p) {
1663                 free(backup_p);
1664                 sq->outnet->svcd_overhead = 0;
1665         }
1666         verbose(VERB_ALGO, "svcd callbacks end");
1667         log_assert(sq->cblist == NULL);
1668         serviced_delete(sq);
1669 }
1670
1671 int
1672 serviced_tcp_callback(struct comm_point* c, void* arg, int error,
1673         struct comm_reply* rep)
1674 {
1675         struct serviced_query* sq = (struct serviced_query*)arg;
1676         struct comm_reply r2;
1677         sq->pending = NULL; /* removed after this callback */
1678         if(error != NETEVENT_NOERROR)
1679                 log_addr(VERB_QUERY, "tcp error for address",
1680                         &sq->addr, sq->addrlen);
1681         if(error==NETEVENT_NOERROR)
1682                 infra_update_tcp_works(sq->outnet->infra, &sq->addr,
1683                         sq->addrlen, sq->zone, sq->zonelen);
1684 #ifdef USE_DNSTAP
1685         if(error==NETEVENT_NOERROR && sq->outnet->dtenv &&
1686            (sq->outnet->dtenv->log_resolver_response_messages ||
1687             sq->outnet->dtenv->log_forwarder_response_messages))
1688                 dt_msg_send_outside_response(sq->outnet->dtenv, &sq->addr,
1689                 c->type, sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1690                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1691 #endif
1692         if(error==NETEVENT_NOERROR && sq->status == serviced_query_TCP_EDNS &&
1693                 (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1694                 LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(sldns_buffer_begin(
1695                 c->buffer)) == LDNS_RCODE_NOTIMPL) ) {
1696                 /* attempt to fallback to nonEDNS */
1697                 sq->status = serviced_query_TCP_EDNS_fallback;
1698                 serviced_tcp_initiate(sq, c->buffer);
1699                 return 0;
1700         } else if(error==NETEVENT_NOERROR &&
1701                 sq->status == serviced_query_TCP_EDNS_fallback &&
1702                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1703                         LDNS_RCODE_NOERROR || LDNS_RCODE_WIRE(
1704                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NXDOMAIN
1705                         || LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1706                         == LDNS_RCODE_YXDOMAIN)) {
1707                 /* the fallback produced a result that looks promising, note
1708                  * that this server should be approached without EDNS */
1709                 /* only store noEDNS in cache if domain is noDNSSEC */
1710                 if(!sq->want_dnssec)
1711                   if(!infra_edns_update(sq->outnet->infra, &sq->addr,
1712                         sq->addrlen, sq->zone, sq->zonelen, -1,
1713                         *sq->outnet->now_secs))
1714                         log_err("Out of memory caching no edns for host");
1715                 sq->status = serviced_query_TCP;
1716         }
1717         if(sq->tcp_upstream || sq->ssl_upstream) {
1718             struct timeval now = *sq->outnet->now_tv;
1719             if(now.tv_sec > sq->last_sent_time.tv_sec ||
1720                 (now.tv_sec == sq->last_sent_time.tv_sec &&
1721                 now.tv_usec > sq->last_sent_time.tv_usec)) {
1722                 /* convert from microseconds to milliseconds */
1723                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1724                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1725                 verbose(VERB_ALGO, "measured TCP-time at %d msec", roundtime);
1726                 log_assert(roundtime >= 0);
1727                 /* only store if less then AUTH_TIMEOUT seconds, it could be
1728                  * huge due to system-hibernated and we woke up */
1729                 if(roundtime < TCP_AUTH_QUERY_TIMEOUT*1000) {
1730                     if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
1731                         sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
1732                         roundtime, sq->last_rtt, (time_t)now.tv_sec))
1733                         log_err("out of memory noting rtt.");
1734                 }
1735             }
1736         }
1737         /* insert address into reply info */
1738         if(!rep) {
1739                 /* create one if there isn't (on errors) */
1740                 rep = &r2;
1741                 r2.c = c;
1742         }
1743         memcpy(&rep->addr, &sq->addr, sq->addrlen);
1744         rep->addrlen = sq->addrlen;
1745         serviced_callbacks(sq, error, c, rep);
1746         return 0;
1747 }
1748
1749 static void
1750 serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
1751 {
1752         verbose(VERB_ALGO, "initiate TCP query %s",
1753                 sq->status==serviced_query_TCP_EDNS?"EDNS":"");
1754         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1755         sq->last_sent_time = *sq->outnet->now_tv;
1756         sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1757                 serviced_tcp_callback, sq);
1758         if(!sq->pending) {
1759                 /* delete from tree so that a retry by above layer does not
1760                  * clash with this entry */
1761                 log_err("serviced_tcp_initiate: failed to send tcp query");
1762                 serviced_callbacks(sq, NETEVENT_CLOSED, NULL, NULL);
1763         }
1764 }
1765
1766 /** Send serviced query over TCP return false on initial failure */
1767 static int
1768 serviced_tcp_send(struct serviced_query* sq, sldns_buffer* buff)
1769 {
1770         int vs, rtt;
1771         uint8_t edns_lame_known;
1772         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1773                 sq->zonelen, *sq->outnet->now_secs, &vs, &edns_lame_known,
1774                 &rtt))
1775                 return 0;
1776         if(vs != -1)
1777                 sq->status = serviced_query_TCP_EDNS;
1778         else    sq->status = serviced_query_TCP;
1779         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1780         sq->last_sent_time = *sq->outnet->now_tv;
1781         sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1782                 serviced_tcp_callback, sq);
1783         return sq->pending != NULL;
1784 }
1785
1786 /* see if packet is edns malformed; got zeroes at start.
1787  * This is from servers that return malformed packets to EDNS0 queries,
1788  * but they return good packets for nonEDNS0 queries.
1789  * We try to detect their output; without resorting to a full parse or
1790  * check for too many bytes after the end of the packet. */
1791 static int
1792 packet_edns_malformed(struct sldns_buffer* buf, int qtype)
1793 {
1794         size_t len;
1795         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE)
1796                 return 1; /* malformed */
1797         /* they have NOERROR rcode, 1 answer. */
1798         if(LDNS_RCODE_WIRE(sldns_buffer_begin(buf)) != LDNS_RCODE_NOERROR)
1799                 return 0;
1800         /* one query (to skip) and answer records */
1801         if(LDNS_QDCOUNT(sldns_buffer_begin(buf)) != 1 ||
1802                 LDNS_ANCOUNT(sldns_buffer_begin(buf)) == 0)
1803                 return 0;
1804         /* skip qname */
1805         len = dname_valid(sldns_buffer_at(buf, LDNS_HEADER_SIZE),
1806                 sldns_buffer_limit(buf)-LDNS_HEADER_SIZE);
1807         if(len == 0)
1808                 return 0;
1809         if(len == 1 && qtype == 0)
1810                 return 0; /* we asked for '.' and type 0 */
1811         /* and then 4 bytes (type and class of query) */
1812         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE + len + 4 + 3)
1813                 return 0;
1814
1815         /* and start with 11 zeroes as the answer RR */
1816         /* so check the qtype of the answer record, qname=0, type=0 */
1817         if(sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[0] == 0 &&
1818            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[1] == 0 &&
1819            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[2] == 0)
1820                 return 1;
1821         return 0;
1822 }
1823
1824 int
1825 serviced_udp_callback(struct comm_point* c, void* arg, int error,
1826         struct comm_reply* rep)
1827 {
1828         struct serviced_query* sq = (struct serviced_query*)arg;
1829         struct outside_network* outnet = sq->outnet;
1830         struct timeval now = *sq->outnet->now_tv;
1831         int fallback_tcp = 0;
1832
1833         sq->pending = NULL; /* removed after callback */
1834         if(error == NETEVENT_TIMEOUT) {
1835                 int rto = 0;
1836                 if(sq->status == serviced_query_PROBE_EDNS) {
1837                         /* non-EDNS probe failed; we do not know its status,
1838                          * keep trying with EDNS, timeout may not be caused
1839                          * by EDNS. */
1840                         sq->status = serviced_query_UDP_EDNS;
1841                 }
1842                 if(sq->status == serviced_query_UDP_EDNS && sq->last_rtt < 5000) {
1843                         /* fallback to 1480/1280 */
1844                         sq->status = serviced_query_UDP_EDNS_FRAG;
1845                         log_name_addr(VERB_ALGO, "try edns1xx0", sq->qbuf+10,
1846                                 &sq->addr, sq->addrlen);
1847                         if(!serviced_udp_send(sq, c->buffer)) {
1848                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1849                         }
1850                         return 0;
1851                 }
1852                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1853                         /* fragmentation size did not fix it */
1854                         sq->status = serviced_query_UDP_EDNS;
1855                 }
1856                 sq->retry++;
1857                 if(!(rto=infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1858                         sq->zone, sq->zonelen, sq->qtype, -1, sq->last_rtt,
1859                         (time_t)now.tv_sec)))
1860                         log_err("out of memory in UDP exponential backoff");
1861                 if(sq->retry < OUTBOUND_UDP_RETRY) {
1862                         log_name_addr(VERB_ALGO, "retry query", sq->qbuf+10,
1863                                 &sq->addr, sq->addrlen);
1864                         if(!serviced_udp_send(sq, c->buffer)) {
1865                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1866                         }
1867                         return 0;
1868                 }
1869                 if(rto >= RTT_MAX_TIMEOUT) {
1870                         fallback_tcp = 1;
1871                         /* UDP does not work, fallback to TCP below */
1872                 } else {
1873                         serviced_callbacks(sq, NETEVENT_TIMEOUT, c, rep);
1874                         return 0;
1875                 }
1876         } else if(error != NETEVENT_NOERROR) {
1877                 /* udp returns error (due to no ID or interface available) */
1878                 serviced_callbacks(sq, error, c, rep);
1879                 return 0;
1880         }
1881 #ifdef USE_DNSTAP
1882         if(error == NETEVENT_NOERROR && outnet->dtenv &&
1883            (outnet->dtenv->log_resolver_response_messages ||
1884             outnet->dtenv->log_forwarder_response_messages))
1885                 dt_msg_send_outside_response(outnet->dtenv, &sq->addr, c->type,
1886                 sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1887                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1888 #endif
1889         if(!fallback_tcp) {
1890             if( (sq->status == serviced_query_UDP_EDNS
1891                 ||sq->status == serviced_query_UDP_EDNS_FRAG)
1892                 && (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1893                         == LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(
1894                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOTIMPL
1895                     || packet_edns_malformed(c->buffer, sq->qtype)
1896                         )) {
1897                 /* try to get an answer by falling back without EDNS */
1898                 verbose(VERB_ALGO, "serviced query: attempt without EDNS");
1899                 sq->status = serviced_query_UDP_EDNS_fallback;
1900                 sq->retry = 0;
1901                 if(!serviced_udp_send(sq, c->buffer)) {
1902                         serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1903                 }
1904                 return 0;
1905             } else if(sq->status == serviced_query_PROBE_EDNS) {
1906                 /* probe without EDNS succeeds, so we conclude that this
1907                  * host likely has EDNS packets dropped */
1908                 log_addr(VERB_DETAIL, "timeouts, concluded that connection to "
1909                         "host drops EDNS packets", &sq->addr, sq->addrlen);
1910                 /* only store noEDNS in cache if domain is noDNSSEC */
1911                 if(!sq->want_dnssec)
1912                   if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1913                         sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
1914                         log_err("Out of memory caching no edns for host");
1915                   }
1916                 sq->status = serviced_query_UDP;
1917             } else if(sq->status == serviced_query_UDP_EDNS &&
1918                 !sq->edns_lame_known) {
1919                 /* now we know that edns queries received answers store that */
1920                 log_addr(VERB_ALGO, "serviced query: EDNS works for",
1921                         &sq->addr, sq->addrlen);
1922                 if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1923                         sq->zone, sq->zonelen, 0, (time_t)now.tv_sec)) {
1924                         log_err("Out of memory caching edns works");
1925                 }
1926                 sq->edns_lame_known = 1;
1927             } else if(sq->status == serviced_query_UDP_EDNS_fallback &&
1928                 !sq->edns_lame_known && (LDNS_RCODE_WIRE(
1929                 sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOERROR ||
1930                 LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1931                 LDNS_RCODE_NXDOMAIN || LDNS_RCODE_WIRE(sldns_buffer_begin(
1932                 c->buffer)) == LDNS_RCODE_YXDOMAIN)) {
1933                 /* the fallback produced a result that looks promising, note
1934                  * that this server should be approached without EDNS */
1935                 /* only store noEDNS in cache if domain is noDNSSEC */
1936                 if(!sq->want_dnssec) {
1937                   log_addr(VERB_ALGO, "serviced query: EDNS fails for",
1938                         &sq->addr, sq->addrlen);
1939                   if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1940                         sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
1941                         log_err("Out of memory caching no edns for host");
1942                   }
1943                 } else {
1944                   log_addr(VERB_ALGO, "serviced query: EDNS fails, but "
1945                         "not stored because need DNSSEC for", &sq->addr,
1946                         sq->addrlen);
1947                 }
1948                 sq->status = serviced_query_UDP;
1949             }
1950             if(now.tv_sec > sq->last_sent_time.tv_sec ||
1951                 (now.tv_sec == sq->last_sent_time.tv_sec &&
1952                 now.tv_usec > sq->last_sent_time.tv_usec)) {
1953                 /* convert from microseconds to milliseconds */
1954                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1955                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1956                 verbose(VERB_ALGO, "measured roundtrip at %d msec", roundtime);
1957                 log_assert(roundtime >= 0);
1958                 /* in case the system hibernated, do not enter a huge value,
1959                  * above this value gives trouble with server selection */
1960                 if(roundtime < 60000) {
1961                     if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1962                         sq->zone, sq->zonelen, sq->qtype, roundtime,
1963                         sq->last_rtt, (time_t)now.tv_sec))
1964                         log_err("out of memory noting rtt.");
1965                 }
1966             }
1967         } /* end of if_!fallback_tcp */
1968         /* perform TC flag check and TCP fallback after updating our
1969          * cache entries for EDNS status and RTT times */
1970         if(LDNS_TC_WIRE(sldns_buffer_begin(c->buffer)) || fallback_tcp) {
1971                 /* fallback to TCP */
1972                 /* this discards partial UDP contents */
1973                 if(sq->status == serviced_query_UDP_EDNS ||
1974                         sq->status == serviced_query_UDP_EDNS_FRAG ||
1975                         sq->status == serviced_query_UDP_EDNS_fallback)
1976                         /* if we have unfinished EDNS_fallback, start again */
1977                         sq->status = serviced_query_TCP_EDNS;
1978                 else    sq->status = serviced_query_TCP;
1979                 serviced_tcp_initiate(sq, c->buffer);
1980                 return 0;
1981         }
1982         /* yay! an answer */
1983         serviced_callbacks(sq, error, c, rep);
1984         return 0;
1985 }
1986
1987 struct serviced_query*
1988 outnet_serviced_query(struct outside_network* outnet,
1989         struct query_info* qinfo, uint16_t flags, int dnssec, int want_dnssec,
1990         int nocaps, int tcp_upstream, int ssl_upstream,
1991         struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
1992         size_t zonelen, struct module_qstate* qstate,
1993         comm_point_callback_t* callback, void* callback_arg, sldns_buffer* buff,
1994         struct module_env* env)
1995 {
1996         struct serviced_query* sq;
1997         struct service_callback* cb;
1998         if(!inplace_cb_query_call(env, qinfo, flags, addr, addrlen, zone, zonelen,
1999                 qstate, qstate->region))
2000                         return NULL;
2001         serviced_gen_query(buff, qinfo->qname, qinfo->qname_len, qinfo->qtype,
2002                 qinfo->qclass, flags);
2003         sq = lookup_serviced(outnet, buff, dnssec, addr, addrlen,
2004                 qstate->edns_opts_back_out);
2005         /* duplicate entries are included in the callback list, because
2006          * there is a counterpart registration by our caller that needs to
2007          * be doubly-removed (with callbacks perhaps). */
2008         if(!(cb = (struct service_callback*)malloc(sizeof(*cb))))
2009                 return NULL;
2010         if(!sq) {
2011                 /* make new serviced query entry */
2012                 sq = serviced_create(outnet, buff, dnssec, want_dnssec, nocaps,
2013                         tcp_upstream, ssl_upstream, addr, addrlen, zone,
2014                         zonelen, (int)qinfo->qtype, qstate->edns_opts_back_out);
2015                 if(!sq) {
2016                         free(cb);
2017                         return NULL;
2018                 }
2019                 /* perform first network action */
2020                 if(outnet->do_udp && !(tcp_upstream || ssl_upstream)) {
2021                         if(!serviced_udp_send(sq, buff)) {
2022                                 (void)rbtree_delete(outnet->serviced, sq);
2023                                 free(sq->qbuf);
2024                                 free(sq->zone);
2025                                 free(sq);
2026                                 free(cb);
2027                                 return NULL;
2028                         }
2029                 } else {
2030                         if(!serviced_tcp_send(sq, buff)) {
2031                                 (void)rbtree_delete(outnet->serviced, sq);
2032                                 free(sq->qbuf);
2033                                 free(sq->zone);
2034                                 free(sq);
2035                                 free(cb);
2036                                 return NULL;
2037                         }
2038                 }
2039         }
2040         /* add callback to list of callbacks */
2041         cb->cb = callback;
2042         cb->cb_arg = callback_arg;
2043         cb->next = sq->cblist;
2044         sq->cblist = cb;
2045         return sq;
2046 }
2047
2048 /** remove callback from list */
2049 static void
2050 callback_list_remove(struct serviced_query* sq, void* cb_arg)
2051 {
2052         struct service_callback** pp = &sq->cblist;
2053         while(*pp) {
2054                 if((*pp)->cb_arg == cb_arg) {
2055                         struct service_callback* del = *pp;
2056                         *pp = del->next;
2057                         free(del);
2058                         return;
2059                 }
2060                 pp = &(*pp)->next;
2061         }
2062 }
2063
2064 void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
2065 {
2066         if(!sq)
2067                 return;
2068         callback_list_remove(sq, cb_arg);
2069         /* if callbacks() routine scheduled deletion, let it do that */
2070         if(!sq->cblist && !sq->to_be_deleted) {
2071                 (void)rbtree_delete(sq->outnet->serviced, sq);
2072                 serviced_delete(sq);
2073         }
2074 }
2075
2076 /** get memory used by waiting tcp entry (in use or not) */
2077 static size_t
2078 waiting_tcp_get_mem(struct waiting_tcp* w)
2079 {
2080         size_t s;
2081         if(!w) return 0;
2082         s = sizeof(*w) + w->pkt_len;
2083         if(w->timer)
2084                 s += comm_timer_get_mem(w->timer);
2085         return s;
2086 }
2087
2088 /** get memory used by port if */
2089 static size_t
2090 if_get_mem(struct port_if* pif)
2091 {
2092         size_t s;
2093         int i;
2094         s = sizeof(*pif) + sizeof(int)*pif->avail_total +
2095                 sizeof(struct port_comm*)*pif->maxout;
2096         for(i=0; i<pif->inuse; i++)
2097                 s += sizeof(*pif->out[i]) +
2098                         comm_point_get_mem(pif->out[i]->cp);
2099         return s;
2100 }
2101
2102 /** get memory used by waiting udp */
2103 static size_t
2104 waiting_udp_get_mem(struct pending* w)
2105 {
2106         size_t s;
2107         s = sizeof(*w) + comm_timer_get_mem(w->timer) + w->pkt_len;
2108         return s;
2109 }
2110
2111 size_t outnet_get_mem(struct outside_network* outnet)
2112 {
2113         size_t i;
2114         int k;
2115         struct waiting_tcp* w;
2116         struct pending* u;
2117         struct serviced_query* sq;
2118         struct service_callback* sb;
2119         struct port_comm* pc;
2120         size_t s = sizeof(*outnet) + sizeof(*outnet->base) +
2121                 sizeof(*outnet->udp_buff) +
2122                 sldns_buffer_capacity(outnet->udp_buff);
2123         /* second buffer is not ours */
2124         for(pc = outnet->unused_fds; pc; pc = pc->next) {
2125                 s += sizeof(*pc) + comm_point_get_mem(pc->cp);
2126         }
2127         for(k=0; k<outnet->num_ip4; k++)
2128                 s += if_get_mem(&outnet->ip4_ifs[k]);
2129         for(k=0; k<outnet->num_ip6; k++)
2130                 s += if_get_mem(&outnet->ip6_ifs[k]);
2131         for(u=outnet->udp_wait_first; u; u=u->next_waiting)
2132                 s += waiting_udp_get_mem(u);
2133
2134         s += sizeof(struct pending_tcp*)*outnet->num_tcp;
2135         for(i=0; i<outnet->num_tcp; i++) {
2136                 s += sizeof(struct pending_tcp);
2137                 s += comm_point_get_mem(outnet->tcp_conns[i]->c);
2138                 if(outnet->tcp_conns[i]->query)
2139                         s += waiting_tcp_get_mem(outnet->tcp_conns[i]->query);
2140         }
2141         for(w=outnet->tcp_wait_first; w; w = w->next_waiting)
2142                 s += waiting_tcp_get_mem(w);
2143         s += sizeof(*outnet->pending);
2144         s += (sizeof(struct pending) + comm_timer_get_mem(NULL)) *
2145                 outnet->pending->count;
2146         s += sizeof(*outnet->serviced);
2147         s += outnet->svcd_overhead;
2148         RBTREE_FOR(sq, struct serviced_query*, outnet->serviced) {
2149                 s += sizeof(*sq) + sq->qbuflen;
2150                 for(sb = sq->cblist; sb; sb = sb->next)
2151                         s += sizeof(*sb);
2152         }
2153         return s;
2154 }
2155
2156 size_t
2157 serviced_get_mem(struct serviced_query* sq)
2158 {
2159         struct service_callback* sb;
2160         size_t s;
2161         s = sizeof(*sq) + sq->qbuflen;
2162         for(sb = sq->cblist; sb; sb = sb->next)
2163                 s += sizeof(*sb);
2164         if(sq->status == serviced_query_UDP_EDNS ||
2165                 sq->status == serviced_query_UDP ||
2166                 sq->status == serviced_query_PROBE_EDNS ||
2167                 sq->status == serviced_query_UDP_EDNS_FRAG ||
2168                 sq->status == serviced_query_UDP_EDNS_fallback) {
2169                 s += sizeof(struct pending);
2170                 s += comm_timer_get_mem(NULL);
2171         } else {
2172                 /* does not have size of the pkt pointer */
2173                 /* always has a timer except on malloc failures */
2174
2175                 /* these sizes are part of the main outside network mem */
2176                 /*
2177                 s += sizeof(struct waiting_tcp);
2178                 s += comm_timer_get_mem(NULL);
2179                 */
2180         }
2181         return s;
2182 }
2183