services/outside_network.c

   1 /*
   2  * services/outside_network.c - implement sending of queries and wait answer.
   3  *
   4  * Copyright (c) 2007, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file has functions to send queries to authoritative servers and
  40  * wait for the pending answer events.
  41  */
  42 #include "config.h"
  43 #include <ctype.h>
  44 #ifdef HAVE_SYS_TYPES_H
  45 #  include <sys/types.h>
  46 #endif
  47 #include <sys/time.h>
  48 #include "services/outside_network.h"
  49 #include "services/listen_dnsport.h"
  50 #include "services/cache/infra.h"
  51 #include "util/data/msgparse.h"
  52 #include "util/data/msgreply.h"
  53 #include "util/data/msgencode.h"
  54 #include "util/data/dname.h"
  55 #include "util/netevent.h"
  56 #include "util/log.h"
  57 #include "util/net_help.h"
  58 #include "util/random.h"
  59 #include "util/fptr_wlist.h"
  60 #include "sldns/sbuffer.h"
  61 #include "dnstap/dnstap.h"
  62 #ifdef HAVE_OPENSSL_SSL_H
  63 #include <openssl/ssl.h>
  64 #endif
  65
  66 #ifdef HAVE_NETDB_H
  67 #include <netdb.h>
  68 #endif
  69 #include <fcntl.h>
  70
  71 /** number of times to retry making a random ID that is unique. */
  72 #define MAX_ID_RETRY 1000
  73 /** number of times to retry finding interface, port that can be opened. */
  74 #define MAX_PORT_RETRY 10000
  75 /** number of retries on outgoing UDP queries */
  76 #define OUTBOUND_UDP_RETRY 1
  77
  78 /** initiate TCP transaction for serviced query */
  79 static void serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff);
  80 /** with a fd available, randomize and send UDP */
  81 static int randomize_and_send_udp(struct pending* pend, sldns_buffer* packet,
  82         int timeout);
  83
  84 /** remove waiting tcp from the outnet waiting list */
  85 static void waiting_list_remove(struct outside_network* outnet,
  86         struct waiting_tcp* w);
  87
  88 int
  89 pending_cmp(const void* key1, const void* key2)
  90 {
  91         struct pending *p1 = (struct pending*)key1;
  92         struct pending *p2 = (struct pending*)key2;
  93         if(p1->id < p2->id)
  94                 return -1;
  95         if(p1->id > p2->id)
  96                 return 1;
  97         log_assert(p1->id == p2->id);
  98         return sockaddr_cmp(&p1->addr, p1->addrlen, &p2->addr, p2->addrlen);
  99 }
 100
 101 int
 102 serviced_cmp(const void* key1, const void* key2)
 103 {
 104         struct serviced_query* q1 = (struct serviced_query*)key1;
 105         struct serviced_query* q2 = (struct serviced_query*)key2;
 106         int r;
 107         if(q1->qbuflen < q2->qbuflen)
 108                 return -1;
 109         if(q1->qbuflen > q2->qbuflen)
 110                 return 1;
 111         log_assert(q1->qbuflen == q2->qbuflen);
 112         log_assert(q1->qbuflen >= 15 /* 10 header, root, type, class */);
 113         /* alternate casing of qname is still the same query */
 114         if((r = memcmp(q1->qbuf, q2->qbuf, 10)) != 0)
 115                 return r;
 116         if((r = memcmp(q1->qbuf+q1->qbuflen-4, q2->qbuf+q2->qbuflen-4, 4)) != 0)
 117                 return r;
 118         if(q1->dnssec != q2->dnssec) {
 119                 if(q1->dnssec < q2->dnssec)
 120                         return -1;
 121                 return 1;
 122         }
 123         if((r = query_dname_compare(q1->qbuf+10, q2->qbuf+10)) != 0)
 124                 return r;
 125         if((r = edns_opt_list_compare(q1->opt_list, q2->opt_list)) != 0)
 126                 return r;
 127         return sockaddr_cmp(&q1->addr, q1->addrlen, &q2->addr, q2->addrlen);
 128 }
 129
 130 /** delete waiting_tcp entry. Does not unlink from waiting list.
 131  * @param w: to delete.
 132  */
 133 static void
 134 waiting_tcp_delete(struct waiting_tcp* w)
 135 {
 136         if(!w) return;
 137         if(w->timer)
 138                 comm_timer_delete(w->timer);
 139         free(w);
 140 }
 141
 142 /**
 143  * Pick random outgoing-interface of that family, and bind it.
 144  * port set to 0 so OS picks a port number for us.
 145  * if it is the ANY address, do not bind.
 146  * @param w: tcp structure with destination address.
 147  * @param s: socket fd.
 148  * @return false on error, socket closed.
 149  */
 150 static int
 151 pick_outgoing_tcp(struct waiting_tcp* w, int s)
 152 {
 153         struct port_if* pi = NULL;
 154         int num;
 155 #ifdef INET6
 156         if(addr_is_ip6(&w->addr, w->addrlen))
 157                 num = w->outnet->num_ip6;
 158         else
 159 #endif
 160                 num = w->outnet->num_ip4;
 161         if(num == 0) {
 162                 log_err("no TCP outgoing interfaces of family");
 163                 log_addr(VERB_OPS, "for addr", &w->addr, w->addrlen);
 164 #ifndef USE_WINSOCK
 165                 close(s);
 166 #else
 167                 closesocket(s);
 168 #endif
 169                 return 0;
 170         }
 171 #ifdef INET6
 172         if(addr_is_ip6(&w->addr, w->addrlen))
 173                 pi = &w->outnet->ip6_ifs[ub_random_max(w->outnet->rnd, num)];
 174         else
 175 #endif
 176                 pi = &w->outnet->ip4_ifs[ub_random_max(w->outnet->rnd, num)];
 177         log_assert(pi);
 178         if(addr_is_any(&pi->addr, pi->addrlen)) {
 179                 /* binding to the ANY interface is for listening sockets */
 180                 return 1;
 181         }
 182         /* set port to 0 */
 183         if(addr_is_ip6(&pi->addr, pi->addrlen))
 184                 ((struct sockaddr_in6*)&pi->addr)->sin6_port = 0;
 185         else    ((struct sockaddr_in*)&pi->addr)->sin_port = 0;
 186         if(bind(s, (struct sockaddr*)&pi->addr, pi->addrlen) != 0) {
 187 #ifndef USE_WINSOCK
 188                 log_err("outgoing tcp: bind: %s", strerror(errno));
 189                 close(s);
 190 #else
 191                 log_err("outgoing tcp: bind: %s",
 192                         wsa_strerror(WSAGetLastError()));
 193                 closesocket(s);
 194 #endif
 195                 return 0;
 196         }
 197         log_addr(VERB_ALGO, "tcp bound to src", &pi->addr, pi->addrlen);
 198         return 1;
 199 }
 200
 201 /** use next free buffer to service a tcp query */
 202 static int
 203 outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 204 {
 205         struct pending_tcp* pend = w->outnet->tcp_free;
 206         int s;
 207 #ifdef SO_REUSEADDR
 208         int on = 1;
 209 #endif
 210         log_assert(pend);
 211         log_assert(pkt);
 212         log_assert(w->addrlen > 0);
 213         /* open socket */
 214 #ifdef INET6
 215         if(addr_is_ip6(&w->addr, w->addrlen))
 216                 s = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
 217         else
 218 #endif
 219                 s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 220         if(s == -1) {
 221 #ifndef USE_WINSOCK
 222                 log_err_addr("outgoing tcp: socket", strerror(errno),
 223                         &w->addr, w->addrlen);
 224 #else
 225                 log_err_addr("outgoing tcp: socket",
 226                         wsa_strerror(WSAGetLastError()), &w->addr, w->addrlen);
 227 #endif
 228                 return 0;
 229         }
 230
 231 #ifdef SO_REUSEADDR
 232         if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on,
 233                 (socklen_t)sizeof(on)) < 0) {
 234                 verbose(VERB_ALGO, "outgoing tcp:"
 235                         " setsockopt(.. SO_REUSEADDR ..) failed");
 236         }
 237 #endif
 238         if (w->outnet->tcp_mss > 0) {
 239 #if defined(IPPROTO_TCP) && defined(TCP_MAXSEG)
 240                 if(setsockopt(s, IPPROTO_TCP, TCP_MAXSEG,
 241                         (void*)&w->outnet->tcp_mss,
 242                         (socklen_t)sizeof(w->outnet->tcp_mss)) < 0) {
 243                         verbose(VERB_ALGO, "outgoing tcp:"
 244                                 " setsockopt(.. TCP_MAXSEG ..) failed");
 245                 }
 246 #else
 247                 verbose(VERB_ALGO, "outgoing tcp:"
 248                         " setsockopt(TCP_MAXSEG) unsupported");
 249 #endif /* defined(IPPROTO_TCP) && defined(TCP_MAXSEG) */
 250         }
 251
 252         if(!pick_outgoing_tcp(w, s))
 253                 return 0;
 254
 255         fd_set_nonblock(s);
 256 #ifdef USE_OSX_MSG_FASTOPEN
 257         /* API for fast open is different here. We use a connectx() function and
 258            then writes can happen as normal even using SSL.*/
 259         /* connectx requires that the len be set in the sockaddr struct*/
 260         struct sockaddr_in *addr_in = (struct sockaddr_in *)&w->addr;
 261         addr_in->sin_len = w->addrlen;
 262         sa_endpoints_t endpoints;
 263         endpoints.sae_srcif = 0;
 264         endpoints.sae_srcaddr = NULL;
 265         endpoints.sae_srcaddrlen = 0;
 266         endpoints.sae_dstaddr = (struct sockaddr *)&w->addr;
 267         endpoints.sae_dstaddrlen = w->addrlen;
 268         if (connectx(s, &endpoints, SAE_ASSOCID_ANY,
 269                      CONNECT_DATA_IDEMPOTENT | CONNECT_RESUME_ON_READ_WRITE,
 270                      NULL, 0, NULL, NULL) == -1) {
 271                 /* if fails, failover to connect for OSX 10.10 */
 272 #ifdef EINPROGRESS
 273                 if(errno != EINPROGRESS) {
 274 #else
 275                 if(1) {
 276 #endif
 277                         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 278 #else /* USE_OSX_MSG_FASTOPEN*/
 279 #ifdef USE_MSG_FASTOPEN
 280         pend->c->tcp_do_fastopen = 1;
 281         /* Only do TFO for TCP in which case no connect() is required here.
 282            Don't combine client TFO with SSL, since OpenSSL can't
 283            currently support doing a handshake on fd that already isn't connected*/
 284         if (w->outnet->sslctx && w->ssl_upstream) {
 285                 if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 286 #else /* USE_MSG_FASTOPEN*/
 287         if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
 288 #endif /* USE_MSG_FASTOPEN*/
 289 #endif /* USE_OSX_MSG_FASTOPEN*/
 290 #ifndef USE_WINSOCK
 291 #ifdef EINPROGRESS
 292                 if(errno != EINPROGRESS) {
 293 #else
 294                 if(1) {
 295 #endif
 296                         if(tcp_connect_errno_needs_log(
 297                                 (struct sockaddr*)&w->addr, w->addrlen))
 298                                 log_err_addr("outgoing tcp: connect",
 299                                         strerror(errno), &w->addr, w->addrlen);
 300                         close(s);
 301 #else /* USE_WINSOCK */
 302                 if(WSAGetLastError() != WSAEINPROGRESS &&
 303                         WSAGetLastError() != WSAEWOULDBLOCK) {
 304                         closesocket(s);
 305 #endif
 306                         return 0;
 307                 }
 308         }
 309 #ifdef USE_MSG_FASTOPEN
 310         }
 311 #endif /* USE_MSG_FASTOPEN */
 312 #ifdef USE_OSX_MSG_FASTOPEN
 313                 }
 314         }
 315 #endif /* USE_OSX_MSG_FASTOPEN */
 316         if(w->outnet->sslctx && w->ssl_upstream) {
 317                 pend->c->ssl = outgoing_ssl_fd(w->outnet->sslctx, s);
 318                 if(!pend->c->ssl) {
 319                         pend->c->fd = s;
 320                         comm_point_close(pend->c);
 321                         return 0;
 322                 }
 323 #ifdef USE_WINSOCK
 324                 comm_point_tcp_win_bio_cb(pend->c, pend->c->ssl);
 325 #endif
 326                 pend->c->ssl_shake_state = comm_ssl_shake_write;
 327         }
 328         w->pkt = NULL;
 329         w->next_waiting = (void*)pend;
 330         pend->id = LDNS_ID_WIRE(pkt);
 331         w->outnet->num_tcp_outgoing++;
 332         w->outnet->tcp_free = pend->next_free;
 333         pend->next_free = NULL;
 334         pend->query = w;
 335         pend->c->repinfo.addrlen = w->addrlen;
 336         memcpy(&pend->c->repinfo.addr, &w->addr, w->addrlen);
 337         sldns_buffer_clear(pend->c->buffer);
 338         sldns_buffer_write(pend->c->buffer, pkt, pkt_len);
 339         sldns_buffer_flip(pend->c->buffer);
 340         pend->c->tcp_is_reading = 0;
 341         pend->c->tcp_byte_count = 0;
 342         comm_point_start_listening(pend->c, s, -1);
 343         return 1;
 344 }
 345
 346 /** see if buffers can be used to service TCP queries */
 347 static void
 348 use_free_buffer(struct outside_network* outnet)
 349 {
 350         struct waiting_tcp* w;
 351         while(outnet->tcp_free && outnet->tcp_wait_first
 352                 && !outnet->want_to_quit) {
 353                 w = outnet->tcp_wait_first;
 354                 outnet->tcp_wait_first = w->next_waiting;
 355                 if(outnet->tcp_wait_last == w)
 356                         outnet->tcp_wait_last = NULL;
 357                 if(!outnet_tcp_take_into_use(w, w->pkt, w->pkt_len)) {
 358                         comm_point_callback_type* cb = w->cb;
 359                         void* cb_arg = w->cb_arg;
 360                         waiting_tcp_delete(w);
 361                         fptr_ok(fptr_whitelist_pending_tcp(cb));
 362                         (void)(*cb)(NULL, cb_arg, NETEVENT_CLOSED, NULL);
 363                 }
 364         }
 365 }
 366
 367 /** decommission a tcp buffer, closes commpoint and frees waiting_tcp entry */
 368 static void
 369 decommission_pending_tcp(struct outside_network* outnet,
 370         struct pending_tcp* pend)
 371 {
 372         if(pend->c->ssl) {
 373 #ifdef HAVE_SSL
 374                 SSL_shutdown(pend->c->ssl);
 375                 SSL_free(pend->c->ssl);
 376                 pend->c->ssl = NULL;
 377 #endif
 378         }
 379         comm_point_close(pend->c);
 380         pend->next_free = outnet->tcp_free;
 381         outnet->tcp_free = pend;
 382         waiting_tcp_delete(pend->query);
 383         pend->query = NULL;
 384         use_free_buffer(outnet);
 385 }
 386
 387 int
 388 outnet_tcp_cb(struct comm_point* c, void* arg, int error,
 389         struct comm_reply *reply_info)
 390 {
 391         struct pending_tcp* pend = (struct pending_tcp*)arg;
 392         struct outside_network* outnet = pend->query->outnet;
 393         verbose(VERB_ALGO, "outnettcp cb");
 394         if(error != NETEVENT_NOERROR) {
 395                 verbose(VERB_QUERY, "outnettcp got tcp error %d", error);
 396                 /* pass error below and exit */
 397         } else {
 398                 /* check ID */
 399                 if(sldns_buffer_limit(c->buffer) < sizeof(uint16_t) ||
 400                         LDNS_ID_WIRE(sldns_buffer_begin(c->buffer))!=pend->id) {
 401                         log_addr(VERB_QUERY,
 402                                 "outnettcp: bad ID in reply, from:",
 403                                 &pend->query->addr, pend->query->addrlen);
 404                         error = NETEVENT_CLOSED;
 405                 }
 406         }
 407         fptr_ok(fptr_whitelist_pending_tcp(pend->query->cb));
 408         (void)(*pend->query->cb)(c, pend->query->cb_arg, error, reply_info);
 409         decommission_pending_tcp(outnet, pend);
 410         return 0;
 411 }
 412
 413 /** lower use count on pc, see if it can be closed */
 414 static void
 415 portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc)
 416 {
 417         struct port_if* pif;
 418         pc->num_outstanding--;
 419         if(pc->num_outstanding > 0) {
 420                 return;
 421         }
 422         /* close it and replace in unused list */
 423         verbose(VERB_ALGO, "close of port %d", pc->number);
 424         comm_point_close(pc->cp);
 425         pif = pc->pif;
 426         log_assert(pif->inuse > 0);
 427         pif->avail_ports[pif->avail_total - pif->inuse] = pc->number;
 428         pif->inuse--;
 429         pif->out[pc->index] = pif->out[pif->inuse];
 430         pif->out[pc->index]->index = pc->index;
 431         pc->next = outnet->unused_fds;
 432         outnet->unused_fds = pc;
 433 }
 434
 435 /** try to send waiting UDP queries */
 436 static void
 437 outnet_send_wait_udp(struct outside_network* outnet)
 438 {
 439         struct pending* pend;
 440         /* process waiting queries */
 441         while(outnet->udp_wait_first && outnet->unused_fds
 442                 && !outnet->want_to_quit) {
 443                 pend = outnet->udp_wait_first;
 444                 outnet->udp_wait_first = pend->next_waiting;
 445                 if(!pend->next_waiting) outnet->udp_wait_last = NULL;
 446                 sldns_buffer_clear(outnet->udp_buff);
 447                 sldns_buffer_write(outnet->udp_buff, pend->pkt, pend->pkt_len);
 448                 sldns_buffer_flip(outnet->udp_buff);
 449                 free(pend->pkt); /* freeing now makes get_mem correct */
 450                 pend->pkt = NULL;
 451                 pend->pkt_len = 0;
 452                 if(!randomize_and_send_udp(pend, outnet->udp_buff,
 453                         pend->timeout)) {
 454                         /* callback error on pending */
 455                         if(pend->cb) {
 456                                 fptr_ok(fptr_whitelist_pending_udp(pend->cb));
 457                                 (void)(*pend->cb)(outnet->unused_fds->cp, pend->cb_arg,
 458                                         NETEVENT_CLOSED, NULL);
 459                         }
 460                         pending_delete(outnet, pend);
 461                 }
 462         }
 463 }
 464
 465 int
 466 outnet_udp_cb(struct comm_point* c, void* arg, int error,
 467         struct comm_reply *reply_info)
 468 {
 469         struct outside_network* outnet = (struct outside_network*)arg;
 470         struct pending key;
 471         struct pending* p;
 472         verbose(VERB_ALGO, "answer cb");
 473
 474         if(error != NETEVENT_NOERROR) {
 475                 verbose(VERB_QUERY, "outnetudp got udp error %d", error);
 476                 return 0;
 477         }
 478         if(sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
 479                 verbose(VERB_QUERY, "outnetudp udp too short");
 480                 return 0;
 481         }
 482         log_assert(reply_info);
 483
 484         /* setup lookup key */
 485         key.id = (unsigned)LDNS_ID_WIRE(sldns_buffer_begin(c->buffer));
 486         memcpy(&key.addr, &reply_info->addr, reply_info->addrlen);
 487         key.addrlen = reply_info->addrlen;
 488         verbose(VERB_ALGO, "Incoming reply id = %4.4x", key.id);
 489         log_addr(VERB_ALGO, "Incoming reply addr =",
 490                 &reply_info->addr, reply_info->addrlen);
 491
 492         /* find it, see if this thing is a valid query response */
 493         verbose(VERB_ALGO, "lookup size is %d entries", (int)outnet->pending->count);
 494         p = (struct pending*)rbtree_search(outnet->pending, &key);
 495         if(!p) {
 496                 verbose(VERB_QUERY, "received unwanted or unsolicited udp reply dropped.");
 497                 log_buf(VERB_ALGO, "dropped message", c->buffer);
 498                 outnet->unwanted_replies++;
 499                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
 500                         >= outnet->unwanted_threshold) {
 501                         log_warn("unwanted reply total reached threshold (%u)"
 502                                 " you may be under attack."
 503                                 " defensive action: clearing the cache",
 504                                 (unsigned)outnet->unwanted_threshold);
 505                         fptr_ok(fptr_whitelist_alloc_cleanup(
 506                                 outnet->unwanted_action));
 507                         (*outnet->unwanted_action)(outnet->unwanted_param);
 508                         outnet->unwanted_total = 0;
 509                 }
 510                 return 0;
 511         }
 512
 513         verbose(VERB_ALGO, "received udp reply.");
 514         log_buf(VERB_ALGO, "udp message", c->buffer);
 515         if(p->pc->cp != c) {
 516                 verbose(VERB_QUERY, "received reply id,addr on wrong port. "
 517                         "dropped.");
 518                 outnet->unwanted_replies++;
 519                 if(outnet->unwanted_threshold && ++outnet->unwanted_total
 520                         >= outnet->unwanted_threshold) {
 521                         log_warn("unwanted reply total reached threshold (%u)"
 522                                 " you may be under attack."
 523                                 " defensive action: clearing the cache",
 524                                 (unsigned)outnet->unwanted_threshold);
 525                         fptr_ok(fptr_whitelist_alloc_cleanup(
 526                                 outnet->unwanted_action));
 527                         (*outnet->unwanted_action)(outnet->unwanted_param);
 528                         outnet->unwanted_total = 0;
 529                 }
 530                 return 0;
 531         }
 532         comm_timer_disable(p->timer);
 533         verbose(VERB_ALGO, "outnet handle udp reply");
 534         /* delete from tree first in case callback creates a retry */
 535         (void)rbtree_delete(outnet->pending, p->node.key);
 536         if(p->cb) {
 537                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
 538                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_NOERROR, reply_info);
 539         }
 540         portcomm_loweruse(outnet, p->pc);
 541         pending_delete(NULL, p);
 542         outnet_send_wait_udp(outnet);
 543         return 0;
 544 }
 545
 546 /** calculate number of ip4 and ip6 interfaces*/
 547 static void
 548 calc_num46(char** ifs, int num_ifs, int do_ip4, int do_ip6,
 549         int* num_ip4, int* num_ip6)
 550 {
 551         int i;
 552         *num_ip4 = 0;
 553         *num_ip6 = 0;
 554         if(num_ifs <= 0) {
 555                 if(do_ip4)
 556                         *num_ip4 = 1;
 557                 if(do_ip6)
 558                         *num_ip6 = 1;
 559                 return;
 560         }
 561         for(i=0; i<num_ifs; i++)
 562         {
 563                 if(str_is_ip6(ifs[i])) {
 564                         if(do_ip6)
 565                                 (*num_ip6)++;
 566                 } else {
 567                         if(do_ip4)
 568                                 (*num_ip4)++;
 569                 }
 570         }
 571
 572 }
 573
 574 void
 575 pending_udp_timer_delay_cb(void* arg)
 576 {
 577         struct pending* p = (struct pending*)arg;
 578         struct outside_network* outnet = p->outnet;
 579         verbose(VERB_ALGO, "timeout udp with delay");
 580         portcomm_loweruse(outnet, p->pc);
 581         pending_delete(outnet, p);
 582         outnet_send_wait_udp(outnet);
 583 }
 584
 585 void
 586 pending_udp_timer_cb(void *arg)
 587 {
 588         struct pending* p = (struct pending*)arg;
 589         struct outside_network* outnet = p->outnet;
 590         /* it timed out */
 591         verbose(VERB_ALGO, "timeout udp");
 592         if(p->cb) {
 593                 fptr_ok(fptr_whitelist_pending_udp(p->cb));
 594                 (void)(*p->cb)(p->pc->cp, p->cb_arg, NETEVENT_TIMEOUT, NULL);
 595         }
 596         /* if delayclose, keep port open for a longer time.
 597          * But if the udpwaitlist exists, then we are struggling to
 598          * keep up with demand for sockets, so do not wait, but service
 599          * the customer (customer service more important than portICMPs) */
 600         if(outnet->delayclose && !outnet->udp_wait_first) {
 601                 p->cb = NULL;
 602                 p->timer->callback = &pending_udp_timer_delay_cb;
 603                 comm_timer_set(p->timer, &outnet->delay_tv);
 604                 return;
 605         }
 606         portcomm_loweruse(outnet, p->pc);
 607         pending_delete(outnet, p);
 608         outnet_send_wait_udp(outnet);
 609 }
 610
 611 /** create pending_tcp buffers */
 612 static int
 613 create_pending_tcp(struct outside_network* outnet, size_t bufsize)
 614 {
 615         size_t i;
 616         if(outnet->num_tcp == 0)
 617                 return 1; /* no tcp needed, nothing to do */
 618         if(!(outnet->tcp_conns = (struct pending_tcp **)calloc(
 619                         outnet->num_tcp, sizeof(struct pending_tcp*))))
 620                 return 0;
 621         for(i=0; i<outnet->num_tcp; i++) {
 622                 if(!(outnet->tcp_conns[i] = (struct pending_tcp*)calloc(1,
 623                         sizeof(struct pending_tcp))))
 624                         return 0;
 625                 outnet->tcp_conns[i]->next_free = outnet->tcp_free;
 626                 outnet->tcp_free = outnet->tcp_conns[i];
 627                 outnet->tcp_conns[i]->c = comm_point_create_tcp_out(
 628                         outnet->base, bufsize, outnet_tcp_cb,
 629                         outnet->tcp_conns[i]);
 630                 if(!outnet->tcp_conns[i]->c)
 631                         return 0;
 632         }
 633         return 1;
 634 }
 635
 636 /** setup an outgoing interface, ready address */
 637 static int setup_if(struct port_if* pif, const char* addrstr,
 638         int* avail, int numavail, size_t numfd)
 639 {
 640         pif->avail_total = numavail;
 641         pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int));
 642         if(!pif->avail_ports)
 643                 return 0;
 644         if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) &&
 645            !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT,
 646                               &pif->addr, &pif->addrlen, &pif->pfxlen))
 647                 return 0;
 648         pif->maxout = (int)numfd;
 649         pif->inuse = 0;
 650         pif->out = (struct port_comm**)calloc(numfd,
 651                 sizeof(struct port_comm*));
 652         if(!pif->out)
 653                 return 0;
 654         return 1;
 655 }
 656
 657 struct outside_network*
 658 outside_network_create(struct comm_base *base, size_t bufsize,
 659         size_t num_ports, char** ifs, int num_ifs, int do_ip4,
 660         int do_ip6, size_t num_tcp, struct infra_cache* infra,
 661         struct ub_randstate* rnd, int use_caps_for_id, int* availports,
 662         int numavailports, size_t unwanted_threshold, int tcp_mss,
 663         void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
 664         void* sslctx, int delayclose, struct dt_env* dtenv)
 665 {
 666         struct outside_network* outnet = (struct outside_network*)
 667                 calloc(1, sizeof(struct outside_network));
 668         size_t k;
 669         if(!outnet) {
 670                 log_err("malloc failed");
 671                 return NULL;
 672         }
 673         comm_base_timept(base, &outnet->now_secs, &outnet->now_tv);
 674         outnet->base = base;
 675         outnet->num_tcp = num_tcp;
 676         outnet->num_tcp_outgoing = 0;
 677         outnet->infra = infra;
 678         outnet->rnd = rnd;
 679         outnet->sslctx = sslctx;
 680 #ifdef USE_DNSTAP
 681         outnet->dtenv = dtenv;
 682 #else
 683         (void)dtenv;
 684 #endif
 685         outnet->svcd_overhead = 0;
 686         outnet->want_to_quit = 0;
 687         outnet->unwanted_threshold = unwanted_threshold;
 688         outnet->unwanted_action = unwanted_action;
 689         outnet->unwanted_param = unwanted_param;
 690         outnet->use_caps_for_id = use_caps_for_id;
 691         outnet->do_udp = do_udp;
 692         outnet->tcp_mss = tcp_mss;
 693 #ifndef S_SPLINT_S
 694         if(delayclose) {
 695                 outnet->delayclose = 1;
 696                 outnet->delay_tv.tv_sec = delayclose/1000;
 697                 outnet->delay_tv.tv_usec = (delayclose%1000)*1000;
 698         }
 699 #endif
 700         if(numavailports == 0) {
 701                 log_err("no outgoing ports available");
 702                 outside_network_delete(outnet);
 703                 return NULL;
 704         }
 705 #ifndef INET6
 706         do_ip6 = 0;
 707 #endif
 708         calc_num46(ifs, num_ifs, do_ip4, do_ip6,
 709                 &outnet->num_ip4, &outnet->num_ip6);
 710         if(outnet->num_ip4 != 0) {
 711                 if(!(outnet->ip4_ifs = (struct port_if*)calloc(
 712                         (size_t)outnet->num_ip4, sizeof(struct port_if)))) {
 713                         log_err("malloc failed");
 714                         outside_network_delete(outnet);
 715                         return NULL;
 716                 }
 717         }
 718         if(outnet->num_ip6 != 0) {
 719                 if(!(outnet->ip6_ifs = (struct port_if*)calloc(
 720                         (size_t)outnet->num_ip6, sizeof(struct port_if)))) {
 721                         log_err("malloc failed");
 722                         outside_network_delete(outnet);
 723                         return NULL;
 724                 }
 725         }
 726         if(     !(outnet->udp_buff = sldns_buffer_new(bufsize)) ||
 727                 !(outnet->pending = rbtree_create(pending_cmp)) ||
 728                 !(outnet->serviced = rbtree_create(serviced_cmp)) ||
 729                 !create_pending_tcp(outnet, bufsize)) {
 730                 log_err("malloc failed");
 731                 outside_network_delete(outnet);
 732                 return NULL;
 733         }
 734
 735         /* allocate commpoints */
 736         for(k=0; k<num_ports; k++) {
 737                 struct port_comm* pc;
 738                 pc = (struct port_comm*)calloc(1, sizeof(*pc));
 739                 if(!pc) {
 740                         log_err("malloc failed");
 741                         outside_network_delete(outnet);
 742                         return NULL;
 743                 }
 744                 pc->cp = comm_point_create_udp(outnet->base, -1,
 745                         outnet->udp_buff, outnet_udp_cb, outnet);
 746                 if(!pc->cp) {
 747                         log_err("malloc failed");
 748                         free(pc);
 749                         outside_network_delete(outnet);
 750                         return NULL;
 751                 }
 752                 pc->next = outnet->unused_fds;
 753                 outnet->unused_fds = pc;
 754         }
 755
 756         /* allocate interfaces */
 757         if(num_ifs == 0) {
 758                 if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0",
 759                         availports, numavailports, num_ports)) {
 760                         log_err("malloc failed");
 761                         outside_network_delete(outnet);
 762                         return NULL;
 763                 }
 764                 if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::",
 765                         availports, numavailports, num_ports)) {
 766                         log_err("malloc failed");
 767                         outside_network_delete(outnet);
 768                         return NULL;
 769                 }
 770         } else {
 771                 size_t done_4 = 0, done_6 = 0;
 772                 int i;
 773                 for(i=0; i<num_ifs; i++) {
 774                         if(str_is_ip6(ifs[i]) && do_ip6) {
 775                                 if(!setup_if(&outnet->ip6_ifs[done_6], ifs[i],
 776                                         availports, numavailports, num_ports)){
 777                                         log_err("malloc failed");
 778                                         outside_network_delete(outnet);
 779                                         return NULL;
 780                                 }
 781                                 done_6++;
 782                         }
 783                         if(!str_is_ip6(ifs[i]) && do_ip4) {
 784                                 if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i],
 785                                         availports, numavailports, num_ports)){
 786                                         log_err("malloc failed");
 787                                         outside_network_delete(outnet);
 788                                         return NULL;
 789                                 }
 790                                 done_4++;
 791                         }
 792                 }
 793         }
 794         return outnet;
 795 }
 796
 797 /** helper pending delete */
 798 static void
 799 pending_node_del(rbnode_type* node, void* arg)
 800 {
 801         struct pending* pend = (struct pending*)node;
 802         struct outside_network* outnet = (struct outside_network*)arg;
 803         pending_delete(outnet, pend);
 804 }
 805
 806 /** helper serviced delete */
 807 static void
 808 serviced_node_del(rbnode_type* node, void* ATTR_UNUSED(arg))
 809 {
 810         struct serviced_query* sq = (struct serviced_query*)node;
 811         struct service_callback* p = sq->cblist, *np;
 812         free(sq->qbuf);
 813         free(sq->zone);
 814         edns_opt_list_free(sq->opt_list);
 815         while(p) {
 816                 np = p->next;
 817                 free(p);
 818                 p = np;
 819         }
 820         free(sq);
 821 }
 822
 823 void
 824 outside_network_quit_prepare(struct outside_network* outnet)
 825 {
 826         if(!outnet)
 827                 return;
 828         /* prevent queued items from being sent */
 829         outnet->want_to_quit = 1;
 830 }
 831
 832 void
 833 outside_network_delete(struct outside_network* outnet)
 834 {
 835         if(!outnet)
 836                 return;
 837         outnet->want_to_quit = 1;
 838         /* check every element, since we can be called on malloc error */
 839         if(outnet->pending) {
 840                 /* free pending elements, but do no unlink from tree. */
 841                 traverse_postorder(outnet->pending, pending_node_del, NULL);
 842                 free(outnet->pending);
 843         }
 844         if(outnet->serviced) {
 845                 traverse_postorder(outnet->serviced, serviced_node_del, NULL);
 846                 free(outnet->serviced);
 847         }
 848         if(outnet->udp_buff)
 849                 sldns_buffer_free(outnet->udp_buff);
 850         if(outnet->unused_fds) {
 851                 struct port_comm* p = outnet->unused_fds, *np;
 852                 while(p) {
 853                         np = p->next;
 854                         comm_point_delete(p->cp);
 855                         free(p);
 856                         p = np;
 857                 }
 858                 outnet->unused_fds = NULL;
 859         }
 860         if(outnet->ip4_ifs) {
 861                 int i, k;
 862                 for(i=0; i<outnet->num_ip4; i++) {
 863                         for(k=0; k<outnet->ip4_ifs[i].inuse; k++) {
 864                                 struct port_comm* pc = outnet->ip4_ifs[i].
 865                                         out[k];
 866                                 comm_point_delete(pc->cp);
 867                                 free(pc);
 868                         }
 869                         free(outnet->ip4_ifs[i].avail_ports);
 870                         free(outnet->ip4_ifs[i].out);
 871                 }
 872                 free(outnet->ip4_ifs);
 873         }
 874         if(outnet->ip6_ifs) {
 875                 int i, k;
 876                 for(i=0; i<outnet->num_ip6; i++) {
 877                         for(k=0; k<outnet->ip6_ifs[i].inuse; k++) {
 878                                 struct port_comm* pc = outnet->ip6_ifs[i].
 879                                         out[k];
 880                                 comm_point_delete(pc->cp);
 881                                 free(pc);
 882                         }
 883                         free(outnet->ip6_ifs[i].avail_ports);
 884                         free(outnet->ip6_ifs[i].out);
 885                 }
 886                 free(outnet->ip6_ifs);
 887         }
 888         if(outnet->tcp_conns) {
 889                 size_t i;
 890                 for(i=0; i<outnet->num_tcp; i++)
 891                         if(outnet->tcp_conns[i]) {
 892                                 comm_point_delete(outnet->tcp_conns[i]->c);
 893                                 waiting_tcp_delete(outnet->tcp_conns[i]->query);
 894                                 free(outnet->tcp_conns[i]);
 895                         }
 896                 free(outnet->tcp_conns);
 897         }
 898         if(outnet->tcp_wait_first) {
 899                 struct waiting_tcp* p = outnet->tcp_wait_first, *np;
 900                 while(p) {
 901                         np = p->next_waiting;
 902                         waiting_tcp_delete(p);
 903                         p = np;
 904                 }
 905         }
 906         if(outnet->udp_wait_first) {
 907                 struct pending* p = outnet->udp_wait_first, *np;
 908                 while(p) {
 909                         np = p->next_waiting;
 910                         pending_delete(NULL, p);
 911                         p = np;
 912                 }
 913         }
 914         free(outnet);
 915 }
 916
 917 void
 918 pending_delete(struct outside_network* outnet, struct pending* p)
 919 {
 920         if(!p)
 921                 return;
 922         if(outnet && outnet->udp_wait_first &&
 923                 (p->next_waiting || p == outnet->udp_wait_last) ) {
 924                 /* delete from waiting list, if it is in the waiting list */
 925                 struct pending* prev = NULL, *x = outnet->udp_wait_first;
 926                 while(x && x != p) {
 927                         prev = x;
 928                         x = x->next_waiting;
 929                 }
 930                 if(x) {
 931                         log_assert(x == p);
 932                         if(prev)
 933                                 prev->next_waiting = p->next_waiting;
 934                         else    outnet->udp_wait_first = p->next_waiting;
 935                         if(outnet->udp_wait_last == p)
 936                                 outnet->udp_wait_last = prev;
 937                 }
 938         }
 939         if(outnet) {
 940                 (void)rbtree_delete(outnet->pending, p->node.key);
 941         }
 942         if(p->timer)
 943                 comm_timer_delete(p->timer);
 944         free(p->pkt);
 945         free(p);
 946 }
 947
 948 static void
 949 sai6_putrandom(struct sockaddr_in6 *sa, int pfxlen, struct ub_randstate *rnd)
 950 {
 951         int i, last;
 952         if(!(pfxlen > 0 && pfxlen < 128))
 953                 return;
 954         for(i = 0; i < (128 - pfxlen) / 8; i++) {
 955                 sa->sin6_addr.s6_addr[15-i] = (uint8_t)ub_random_max(rnd, 256);
 956         }
 957         last = pfxlen & 7;
 958         if(last != 0) {
 959                 sa->sin6_addr.s6_addr[15-i] |=
 960                         ((0xFF >> last) & ub_random_max(rnd, 256));
 961         }
 962 }
 963
 964 /**
 965  * Try to open a UDP socket for outgoing communication.
 966  * Sets sockets options as needed.
 967  * @param addr: socket address.
 968  * @param addrlen: length of address.
 969  * @param pfxlen: length of network prefix (for address randomisation).
 970  * @param port: port override for addr.
 971  * @param inuse: if -1 is returned, this bool means the port was in use.
 972  * @param rnd: random state (for address randomisation).
 973  * @return fd or -1
 974  */
 975 static int
 976 udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen,
 977         int port, int* inuse, struct ub_randstate* rnd)
 978 {
 979         int fd, noproto;
 980         if(addr_is_ip6(addr, addrlen)) {
 981                 int freebind = 0;
 982                 struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr;
 983                 sa.sin6_port = (in_port_t)htons((uint16_t)port);
 984                 if(pfxlen != 0) {
 985                         freebind = 1;
 986                         sai6_putrandom(&sa, pfxlen, rnd);
 987                 }
 988                 fd = create_udp_sock(AF_INET6, SOCK_DGRAM,
 989                         (struct sockaddr*)&sa, addrlen, 1, inuse, &noproto,
 990                         0, 0, 0, NULL, 0, freebind, 0);
 991         } else {
 992                 struct sockaddr_in* sa = (struct sockaddr_in*)addr;
 993                 sa->sin_port = (in_port_t)htons((uint16_t)port);
 994                 fd = create_udp_sock(AF_INET, SOCK_DGRAM,
 995                         (struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
 996                         0, 0, 0, NULL, 0, 0, 0);
 997         }
 998         return fd;
 999 }
1000
1001 /** Select random ID */
1002 static int
1003 select_id(struct outside_network* outnet, struct pending* pend,
1004         sldns_buffer* packet)
1005 {
1006         int id_tries = 0;
1007         pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
1008         LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
1009
1010         /* insert in tree */
1011         pend->node.key = pend;
1012         while(!rbtree_insert(outnet->pending, &pend->node)) {
1013                 /* change ID to avoid collision */
1014                 pend->id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
1015                 LDNS_ID_SET(sldns_buffer_begin(packet), pend->id);
1016                 id_tries++;
1017                 if(id_tries == MAX_ID_RETRY) {
1018                         pend->id=99999; /* non existant ID */
1019                         log_err("failed to generate unique ID, drop msg");
1020                         return 0;
1021                 }
1022         }
1023         verbose(VERB_ALGO, "inserted new pending reply id=%4.4x", pend->id);
1024         return 1;
1025 }
1026
1027 /** Select random interface and port */
1028 static int
1029 select_ifport(struct outside_network* outnet, struct pending* pend,
1030         int num_if, struct port_if* ifs)
1031 {
1032         int my_if, my_port, fd, portno, inuse, tries=0;
1033         struct port_if* pif;
1034         /* randomly select interface and port */
1035         if(num_if == 0) {
1036                 verbose(VERB_QUERY, "Need to send query but have no "
1037                         "outgoing interfaces of that family");
1038                 return 0;
1039         }
1040         log_assert(outnet->unused_fds);
1041         tries = 0;
1042         while(1) {
1043                 my_if = ub_random_max(outnet->rnd, num_if);
1044                 pif = &ifs[my_if];
1045                 my_port = ub_random_max(outnet->rnd, pif->avail_total);
1046                 if(my_port < pif->inuse) {
1047                         /* port already open */
1048                         pend->pc = pif->out[my_port];
1049                         verbose(VERB_ALGO, "using UDP if=%d port=%d",
1050                                 my_if, pend->pc->number);
1051                         break;
1052                 }
1053                 /* try to open new port, if fails, loop to try again */
1054                 log_assert(pif->inuse < pif->maxout);
1055                 portno = pif->avail_ports[my_port - pif->inuse];
1056                 fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen,
1057                         portno, &inuse, outnet->rnd);
1058                 if(fd == -1 && !inuse) {
1059                         /* nonrecoverable error making socket */
1060                         return 0;
1061                 }
1062                 if(fd != -1) {
1063                         verbose(VERB_ALGO, "opened UDP if=%d port=%d",
1064                                 my_if, portno);
1065                         /* grab fd */
1066                         pend->pc = outnet->unused_fds;
1067                         outnet->unused_fds = pend->pc->next;
1068
1069                         /* setup portcomm */
1070                         pend->pc->next = NULL;
1071                         pend->pc->number = portno;
1072                         pend->pc->pif = pif;
1073                         pend->pc->index = pif->inuse;
1074                         pend->pc->num_outstanding = 0;
1075                         comm_point_start_listening(pend->pc->cp, fd, -1);
1076
1077                         /* grab port in interface */
1078                         pif->out[pif->inuse] = pend->pc;
1079                         pif->avail_ports[my_port - pif->inuse] =
1080                                 pif->avail_ports[pif->avail_total-pif->inuse-1];
1081                         pif->inuse++;
1082                         break;
1083                 }
1084                 /* failed, already in use */
1085                 verbose(VERB_QUERY, "port %d in use, trying another", portno);
1086                 tries++;
1087                 if(tries == MAX_PORT_RETRY) {
1088                         log_err("failed to find an open port, drop msg");
1089                         return 0;
1090                 }
1091         }
1092         log_assert(pend->pc);
1093         pend->pc->num_outstanding++;
1094
1095         return 1;
1096 }
1097
1098 static int
1099 randomize_and_send_udp(struct pending* pend, sldns_buffer* packet, int timeout)
1100 {
1101         struct timeval tv;
1102         struct outside_network* outnet = pend->sq->outnet;
1103
1104         /* select id */
1105         if(!select_id(outnet, pend, packet)) {
1106                 return 0;
1107         }
1108
1109         /* select src_if, port */
1110         if(addr_is_ip6(&pend->addr, pend->addrlen)) {
1111                 if(!select_ifport(outnet, pend,
1112                         outnet->num_ip6, outnet->ip6_ifs))
1113                         return 0;
1114         } else {
1115                 if(!select_ifport(outnet, pend,
1116                         outnet->num_ip4, outnet->ip4_ifs))
1117                         return 0;
1118         }
1119         log_assert(pend->pc && pend->pc->cp);
1120
1121         /* send it over the commlink */
1122         if(!comm_point_send_udp_msg(pend->pc->cp, packet,
1123                 (struct sockaddr*)&pend->addr, pend->addrlen)) {
1124                 portcomm_loweruse(outnet, pend->pc);
1125                 return 0;
1126         }
1127
1128         /* system calls to set timeout after sending UDP to make roundtrip
1129            smaller. */
1130 #ifndef S_SPLINT_S
1131         tv.tv_sec = timeout/1000;
1132         tv.tv_usec = (timeout%1000)*1000;
1133 #endif
1134         comm_timer_set(pend->timer, &tv);
1135
1136 #ifdef USE_DNSTAP
1137         if(outnet->dtenv &&
1138            (outnet->dtenv->log_resolver_query_messages ||
1139             outnet->dtenv->log_forwarder_query_messages))
1140                 dt_msg_send_outside_query(outnet->dtenv, &pend->addr, comm_udp,
1141                 pend->sq->zone, pend->sq->zonelen, packet);
1142 #endif
1143         return 1;
1144 }
1145
1146 struct pending*
1147 pending_udp_query(struct serviced_query* sq, struct sldns_buffer* packet,
1148         int timeout, comm_point_callback_type* cb, void* cb_arg)
1149 {
1150         struct pending* pend = (struct pending*)calloc(1, sizeof(*pend));
1151         if(!pend) return NULL;
1152         pend->outnet = sq->outnet;
1153         pend->sq = sq;
1154         pend->addrlen = sq->addrlen;
1155         memmove(&pend->addr, &sq->addr, sq->addrlen);
1156         pend->cb = cb;
1157         pend->cb_arg = cb_arg;
1158         pend->node.key = pend;
1159         pend->timer = comm_timer_create(sq->outnet->base, pending_udp_timer_cb,
1160                 pend);
1161         if(!pend->timer) {
1162                 free(pend);
1163                 return NULL;
1164         }
1165
1166         if(sq->outnet->unused_fds == NULL) {
1167                 /* no unused fd, cannot create a new port (randomly) */
1168                 verbose(VERB_ALGO, "no fds available, udp query waiting");
1169                 pend->timeout = timeout;
1170                 pend->pkt_len = sldns_buffer_limit(packet);
1171                 pend->pkt = (uint8_t*)memdup(sldns_buffer_begin(packet),
1172                         pend->pkt_len);
1173                 if(!pend->pkt) {
1174                         comm_timer_delete(pend->timer);
1175                         free(pend);
1176                         return NULL;
1177                 }
1178                 /* put at end of waiting list */
1179                 if(sq->outnet->udp_wait_last)
1180                         sq->outnet->udp_wait_last->next_waiting = pend;
1181                 else
1182                         sq->outnet->udp_wait_first = pend;
1183                 sq->outnet->udp_wait_last = pend;
1184                 return pend;
1185         }
1186         if(!randomize_and_send_udp(pend, packet, timeout)) {
1187                 pending_delete(sq->outnet, pend);
1188                 return NULL;
1189         }
1190         return pend;
1191 }
1192
1193 void
1194 outnet_tcptimer(void* arg)
1195 {
1196         struct waiting_tcp* w = (struct waiting_tcp*)arg;
1197         struct outside_network* outnet = w->outnet;
1198         comm_point_callback_type* cb;
1199         void* cb_arg;
1200         if(w->pkt) {
1201                 /* it is on the waiting list */
1202                 waiting_list_remove(outnet, w);
1203         } else {
1204                 /* it was in use */
1205                 struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
1206                 comm_point_close(pend->c);
1207                 pend->query = NULL;
1208                 pend->next_free = outnet->tcp_free;
1209                 outnet->tcp_free = pend;
1210         }
1211         cb = w->cb;
1212         cb_arg = w->cb_arg;
1213         waiting_tcp_delete(w);
1214         fptr_ok(fptr_whitelist_pending_tcp(cb));
1215         (void)(*cb)(NULL, cb_arg, NETEVENT_TIMEOUT, NULL);
1216         use_free_buffer(outnet);
1217 }
1218
1219 struct waiting_tcp*
1220 pending_tcp_query(struct serviced_query* sq, sldns_buffer* packet,
1221         int timeout, comm_point_callback_type* callback, void* callback_arg)
1222 {
1223         struct pending_tcp* pend = sq->outnet->tcp_free;
1224         struct waiting_tcp* w;
1225         struct timeval tv;
1226         uint16_t id;
1227         /* if no buffer is free allocate space to store query */
1228         w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp)
1229                 + (pend?0:sldns_buffer_limit(packet)));
1230         if(!w) {
1231                 return NULL;
1232         }
1233         if(!(w->timer = comm_timer_create(sq->outnet->base, outnet_tcptimer, w))) {
1234                 free(w);
1235                 return NULL;
1236         }
1237         w->pkt = NULL;
1238         w->pkt_len = 0;
1239         id = ((unsigned)ub_random(sq->outnet->rnd)>>8) & 0xffff;
1240         LDNS_ID_SET(sldns_buffer_begin(packet), id);
1241         memcpy(&w->addr, &sq->addr, sq->addrlen);
1242         w->addrlen = sq->addrlen;
1243         w->outnet = sq->outnet;
1244         w->cb = callback;
1245         w->cb_arg = callback_arg;
1246         w->ssl_upstream = sq->ssl_upstream;
1247 #ifndef S_SPLINT_S
1248         tv.tv_sec = timeout;
1249         tv.tv_usec = 0;
1250 #endif
1251         comm_timer_set(w->timer, &tv);
1252         if(pend) {
1253                 /* we have a buffer available right now */
1254                 if(!outnet_tcp_take_into_use(w, sldns_buffer_begin(packet),
1255                         sldns_buffer_limit(packet))) {
1256                         waiting_tcp_delete(w);
1257                         return NULL;
1258                 }
1259 #ifdef USE_DNSTAP
1260                 if(sq->outnet->dtenv &&
1261                    (sq->outnet->dtenv->log_resolver_query_messages ||
1262                     sq->outnet->dtenv->log_forwarder_query_messages))
1263                 dt_msg_send_outside_query(sq->outnet->dtenv, &sq->addr,
1264                 comm_tcp, sq->zone, sq->zonelen, packet);
1265 #endif
1266         } else {
1267                 /* queue up */
1268                 w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
1269                 w->pkt_len = sldns_buffer_limit(packet);
1270                 memmove(w->pkt, sldns_buffer_begin(packet), w->pkt_len);
1271                 w->next_waiting = NULL;
1272                 if(sq->outnet->tcp_wait_last)
1273                         sq->outnet->tcp_wait_last->next_waiting = w;
1274                 else    sq->outnet->tcp_wait_first = w;
1275                 sq->outnet->tcp_wait_last = w;
1276         }
1277         return w;
1278 }
1279
1280 /** create query for serviced queries */
1281 static void
1282 serviced_gen_query(sldns_buffer* buff, uint8_t* qname, size_t qnamelen,
1283         uint16_t qtype, uint16_t qclass, uint16_t flags)
1284 {
1285         sldns_buffer_clear(buff);
1286         /* skip id */
1287         sldns_buffer_write_u16(buff, flags);
1288         sldns_buffer_write_u16(buff, 1); /* qdcount */
1289         sldns_buffer_write_u16(buff, 0); /* ancount */
1290         sldns_buffer_write_u16(buff, 0); /* nscount */
1291         sldns_buffer_write_u16(buff, 0); /* arcount */
1292         sldns_buffer_write(buff, qname, qnamelen);
1293         sldns_buffer_write_u16(buff, qtype);
1294         sldns_buffer_write_u16(buff, qclass);
1295         sldns_buffer_flip(buff);
1296 }
1297
1298 /** lookup serviced query in serviced query rbtree */
1299 static struct serviced_query*
1300 lookup_serviced(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1301         struct sockaddr_storage* addr, socklen_t addrlen,
1302         struct edns_option* opt_list)
1303 {
1304         struct serviced_query key;
1305         key.node.key = &key;
1306         key.qbuf = sldns_buffer_begin(buff);
1307         key.qbuflen = sldns_buffer_limit(buff);
1308         key.dnssec = dnssec;
1309         memcpy(&key.addr, addr, addrlen);
1310         key.addrlen = addrlen;
1311         key.outnet = outnet;
1312         key.opt_list = opt_list;
1313         return (struct serviced_query*)rbtree_search(outnet->serviced, &key);
1314 }
1315
1316 /** Create new serviced entry */
1317 static struct serviced_query*
1318 serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
1319         int want_dnssec, int nocaps, int tcp_upstream, int ssl_upstream,
1320         struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
1321         size_t zonelen, int qtype, struct edns_option* opt_list)
1322 {
1323         struct serviced_query* sq = (struct serviced_query*)malloc(sizeof(*sq));
1324 #ifdef UNBOUND_DEBUG
1325         rbnode_type* ins;
1326 #endif
1327         if(!sq)
1328                 return NULL;
1329         sq->node.key = sq;
1330         sq->qbuf = memdup(sldns_buffer_begin(buff), sldns_buffer_limit(buff));
1331         if(!sq->qbuf) {
1332                 free(sq);
1333                 return NULL;
1334         }
1335         sq->qbuflen = sldns_buffer_limit(buff);
1336         sq->zone = memdup(zone, zonelen);
1337         if(!sq->zone) {
1338                 free(sq->qbuf);
1339                 free(sq);
1340                 return NULL;
1341         }
1342         sq->zonelen = zonelen;
1343         sq->qtype = qtype;
1344         sq->dnssec = dnssec;
1345         sq->want_dnssec = want_dnssec;
1346         sq->nocaps = nocaps;
1347         sq->tcp_upstream = tcp_upstream;
1348         sq->ssl_upstream = ssl_upstream;
1349         memcpy(&sq->addr, addr, addrlen);
1350         sq->addrlen = addrlen;
1351         sq->opt_list = NULL;
1352         if(opt_list) {
1353                 sq->opt_list = edns_opt_copy_alloc(opt_list);
1354                 if(!sq->opt_list) {
1355                         free(sq->zone);
1356                         free(sq->qbuf);
1357                         free(sq);
1358                         return NULL;
1359                 }
1360         }
1361         sq->outnet = outnet;
1362         sq->cblist = NULL;
1363         sq->pending = NULL;
1364         sq->status = serviced_initial;
1365         sq->retry = 0;
1366         sq->to_be_deleted = 0;
1367 #ifdef UNBOUND_DEBUG
1368         ins =
1369 #else
1370         (void)
1371 #endif
1372         rbtree_insert(outnet->serviced, &sq->node);
1373         log_assert(ins != NULL); /* must not be already present */
1374         return sq;
1375 }
1376
1377 /** remove waiting tcp from the outnet waiting list */
1378 static void
1379 waiting_list_remove(struct outside_network* outnet, struct waiting_tcp* w)
1380 {
1381         struct waiting_tcp* p = outnet->tcp_wait_first, *prev = NULL;
1382         while(p) {
1383                 if(p == w) {
1384                         /* remove w */
1385                         if(prev)
1386                                 prev->next_waiting = w->next_waiting;
1387                         else    outnet->tcp_wait_first = w->next_waiting;
1388                         if(outnet->tcp_wait_last == w)
1389                                 outnet->tcp_wait_last = prev;
1390                         return;
1391                 }
1392                 prev = p;
1393                 p = p->next_waiting;
1394         }
1395 }
1396
1397 /** cleanup serviced query entry */
1398 static void
1399 serviced_delete(struct serviced_query* sq)
1400 {
1401         if(sq->pending) {
1402                 /* clear up the pending query */
1403                 if(sq->status == serviced_query_UDP_EDNS ||
1404                         sq->status == serviced_query_UDP ||
1405                         sq->status == serviced_query_PROBE_EDNS ||
1406                         sq->status == serviced_query_UDP_EDNS_FRAG ||
1407                         sq->status == serviced_query_UDP_EDNS_fallback) {
1408                         struct pending* p = (struct pending*)sq->pending;
1409                         if(p->pc)
1410                                 portcomm_loweruse(sq->outnet, p->pc);
1411                         pending_delete(sq->outnet, p);
1412                         /* this call can cause reentrant calls back into the
1413                          * mesh */
1414                         outnet_send_wait_udp(sq->outnet);
1415                 } else {
1416                         struct waiting_tcp* p = (struct waiting_tcp*)
1417                                 sq->pending;
1418                         if(p->pkt == NULL) {
1419                                 decommission_pending_tcp(sq->outnet,
1420                                         (struct pending_tcp*)p->next_waiting);
1421                         } else {
1422                                 waiting_list_remove(sq->outnet, p);
1423                                 waiting_tcp_delete(p);
1424                         }
1425                 }
1426         }
1427         /* does not delete from tree, caller has to do that */
1428         serviced_node_del(&sq->node, NULL);
1429 }
1430
1431 /** perturb a dname capitalization randomly */
1432 static void
1433 serviced_perturb_qname(struct ub_randstate* rnd, uint8_t* qbuf, size_t len)
1434 {
1435         uint8_t lablen;
1436         uint8_t* d = qbuf + 10;
1437         long int random = 0;
1438         int bits = 0;
1439         log_assert(len >= 10 + 5 /* offset qname, root, qtype, qclass */);
1440         (void)len;
1441         lablen = *d++;
1442         while(lablen) {
1443                 while(lablen--) {
1444                         /* only perturb A-Z, a-z */
1445                         if(isalpha((unsigned char)*d)) {
1446                                 /* get a random bit */
1447                                 if(bits == 0) {
1448                                         random = ub_random(rnd);
1449                                         bits = 30;
1450                                 }
1451                                 if(random & 0x1) {
1452                                         *d = (uint8_t)toupper((unsigned char)*d);
1453                                 } else {
1454                                         *d = (uint8_t)tolower((unsigned char)*d);
1455                                 }
1456                                 random >>= 1;
1457                                 bits--;
1458                         }
1459                         d++;
1460                 }
1461                 lablen = *d++;
1462         }
1463         if(verbosity >= VERB_ALGO) {
1464                 char buf[LDNS_MAX_DOMAINLEN+1];
1465                 dname_str(qbuf+10, buf);
1466                 verbose(VERB_ALGO, "qname perturbed to %s", buf);
1467         }
1468 }
1469
1470 /** put serviced query into a buffer */
1471 static void
1472 serviced_encode(struct serviced_query* sq, sldns_buffer* buff, int with_edns)
1473 {
1474         /* if we are using 0x20 bits for ID randomness, perturb them */
1475         if(sq->outnet->use_caps_for_id && !sq->nocaps) {
1476                 serviced_perturb_qname(sq->outnet->rnd, sq->qbuf, sq->qbuflen);
1477         }
1478         /* generate query */
1479         sldns_buffer_clear(buff);
1480         sldns_buffer_write_u16(buff, 0); /* id placeholder */
1481         sldns_buffer_write(buff, sq->qbuf, sq->qbuflen);
1482         sldns_buffer_flip(buff);
1483         if(with_edns) {
1484                 /* add edns section */
1485                 struct edns_data edns;
1486                 edns.edns_present = 1;
1487                 edns.ext_rcode = 0;
1488                 edns.edns_version = EDNS_ADVERTISED_VERSION;
1489                 edns.opt_list = sq->opt_list;
1490                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1491                         if(addr_is_ip6(&sq->addr, sq->addrlen)) {
1492                                 if(EDNS_FRAG_SIZE_IP6 < EDNS_ADVERTISED_SIZE)
1493                                         edns.udp_size = EDNS_FRAG_SIZE_IP6;
1494                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
1495                         } else {
1496                                 if(EDNS_FRAG_SIZE_IP4 < EDNS_ADVERTISED_SIZE)
1497                                         edns.udp_size = EDNS_FRAG_SIZE_IP4;
1498                                 else    edns.udp_size = EDNS_ADVERTISED_SIZE;
1499                         }
1500                 } else {
1501                         edns.udp_size = EDNS_ADVERTISED_SIZE;
1502                 }
1503                 edns.bits = 0;
1504                 if(sq->dnssec & EDNS_DO)
1505                         edns.bits = EDNS_DO;
1506                 if(sq->dnssec & BIT_CD)
1507                         LDNS_CD_SET(sldns_buffer_begin(buff));
1508                 attach_edns_record(buff, &edns);
1509         }
1510 }
1511
1512 /**
1513  * Perform serviced query UDP sending operation.
1514  * Sends UDP with EDNS, unless infra host marked non EDNS.
1515  * @param sq: query to send.
1516  * @param buff: buffer scratch space.
1517  * @return 0 on error.
1518  */
1519 static int
1520 serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
1521 {
1522         int rtt, vs;
1523         uint8_t edns_lame_known;
1524         time_t now = *sq->outnet->now_secs;
1525
1526         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1527                 sq->zonelen, now, &vs, &edns_lame_known, &rtt))
1528                 return 0;
1529         sq->last_rtt = rtt;
1530         verbose(VERB_ALGO, "EDNS lookup known=%d vs=%d", edns_lame_known, vs);
1531         if(sq->status == serviced_initial) {
1532                 if(edns_lame_known == 0 && rtt > 5000 && rtt < 10001) {
1533                         /* perform EDNS lame probe - check if server is
1534                          * EDNS lame (EDNS queries to it are dropped) */
1535                         verbose(VERB_ALGO, "serviced query: send probe to see "
1536                                 " if use of EDNS causes timeouts");
1537                         /* even 700 msec may be too small */
1538                         rtt = 1000;
1539                         sq->status = serviced_query_PROBE_EDNS;
1540                 } else if(vs != -1) {
1541                         sq->status = serviced_query_UDP_EDNS;
1542                 } else {
1543                         sq->status = serviced_query_UDP;
1544                 }
1545         }
1546         serviced_encode(sq, buff, (sq->status == serviced_query_UDP_EDNS) ||
1547                 (sq->status == serviced_query_UDP_EDNS_FRAG));
1548         sq->last_sent_time = *sq->outnet->now_tv;
1549         sq->edns_lame_known = (int)edns_lame_known;
1550         verbose(VERB_ALGO, "serviced query UDP timeout=%d msec", rtt);
1551         sq->pending = pending_udp_query(sq, buff, rtt,
1552                 serviced_udp_callback, sq);
1553         if(!sq->pending)
1554                 return 0;
1555         return 1;
1556 }
1557
1558 /** check that perturbed qname is identical */
1559 static int
1560 serviced_check_qname(sldns_buffer* pkt, uint8_t* qbuf, size_t qbuflen)
1561 {
1562         uint8_t* d1 = sldns_buffer_begin(pkt)+12;
1563         uint8_t* d2 = qbuf+10;
1564         uint8_t len1, len2;
1565         int count = 0;
1566         if(sldns_buffer_limit(pkt) < 12+1+4) /* packet too small for qname */
1567                 return 0;
1568         log_assert(qbuflen >= 15 /* 10 header, root, type, class */);
1569         len1 = *d1++;
1570         len2 = *d2++;
1571         while(len1 != 0 || len2 != 0) {
1572                 if(LABEL_IS_PTR(len1)) {
1573                         /* check if we can read *d1 with compression ptr rest */
1574                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1575                                 return 0;
1576                         d1 = sldns_buffer_begin(pkt)+PTR_OFFSET(len1, *d1);
1577                         /* check if we can read the destination *d1 */
1578                         if(d1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1579                                 return 0;
1580                         len1 = *d1++;
1581                         if(count++ > MAX_COMPRESS_PTRS)
1582                                 return 0;
1583                         continue;
1584                 }
1585                 if(d2 > qbuf+qbuflen)
1586                         return 0;
1587                 if(len1 != len2)
1588                         return 0;
1589                 if(len1 > LDNS_MAX_LABELLEN)
1590                         return 0;
1591                 /* check len1 + 1(next length) are okay to read */
1592                 if(d1+len1 >= sldns_buffer_at(pkt, sldns_buffer_limit(pkt)))
1593                         return 0;
1594                 log_assert(len1 <= LDNS_MAX_LABELLEN);
1595                 log_assert(len2 <= LDNS_MAX_LABELLEN);
1596                 log_assert(len1 == len2 && len1 != 0);
1597                 /* compare the labels - bitwise identical */
1598                 if(memcmp(d1, d2, len1) != 0)
1599                         return 0;
1600                 d1 += len1;
1601                 d2 += len2;
1602                 len1 = *d1++;
1603                 len2 = *d2++;
1604         }
1605         return 1;
1606 }
1607
1608 /** call the callbacks for a serviced query */
1609 static void
1610 serviced_callbacks(struct serviced_query* sq, int error, struct comm_point* c,
1611         struct comm_reply* rep)
1612 {
1613         struct service_callback* p;
1614         int dobackup = (sq->cblist && sq->cblist->next); /* >1 cb*/
1615         uint8_t *backup_p = NULL;
1616         size_t backlen = 0;
1617 #ifdef UNBOUND_DEBUG
1618         rbnode_type* rem =
1619 #else
1620         (void)
1621 #endif
1622         /* remove from tree, and schedule for deletion, so that callbacks
1623          * can safely deregister themselves and even create new serviced
1624          * queries that are identical to this one. */
1625         rbtree_delete(sq->outnet->serviced, sq);
1626         log_assert(rem); /* should have been present */
1627         sq->to_be_deleted = 1;
1628         verbose(VERB_ALGO, "svcd callbacks start");
1629         if(sq->outnet->use_caps_for_id && error == NETEVENT_NOERROR && c &&
1630                 !sq->nocaps && sq->qtype != LDNS_RR_TYPE_PTR) {
1631                 /* for type PTR do not check perturbed name in answer,
1632                  * compatibility with cisco dns guard boxes that mess up
1633                  * reverse queries 0x20 contents */
1634                 /* noerror and nxdomain must have a qname in reply */
1635                 if(sldns_buffer_read_u16_at(c->buffer, 4) == 0 &&
1636                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1637                                 == LDNS_RCODE_NOERROR ||
1638                          LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1639                                 == LDNS_RCODE_NXDOMAIN)) {
1640                         verbose(VERB_DETAIL, "no qname in reply to check 0x20ID");
1641                         log_addr(VERB_DETAIL, "from server",
1642                                 &sq->addr, sq->addrlen);
1643                         log_buf(VERB_DETAIL, "for packet", c->buffer);
1644                         error = NETEVENT_CLOSED;
1645                         c = NULL;
1646                 } else if(sldns_buffer_read_u16_at(c->buffer, 4) > 0 &&
1647                         !serviced_check_qname(c->buffer, sq->qbuf,
1648                         sq->qbuflen)) {
1649                         verbose(VERB_DETAIL, "wrong 0x20-ID in reply qname");
1650                         log_addr(VERB_DETAIL, "from server",
1651                                 &sq->addr, sq->addrlen);
1652                         log_buf(VERB_DETAIL, "for packet", c->buffer);
1653                         error = NETEVENT_CAPSFAIL;
1654                         /* and cleanup too */
1655                         pkt_dname_tolower(c->buffer,
1656                                 sldns_buffer_at(c->buffer, 12));
1657                 } else {
1658                         verbose(VERB_ALGO, "good 0x20-ID in reply qname");
1659                         /* cleanup caps, prettier cache contents. */
1660                         pkt_dname_tolower(c->buffer,
1661                                 sldns_buffer_at(c->buffer, 12));
1662                 }
1663         }
1664         if(dobackup && c) {
1665                 /* make a backup of the query, since the querystate processing
1666                  * may send outgoing queries that overwrite the buffer.
1667                  * use secondary buffer to store the query.
1668                  * This is a data copy, but faster than packet to server */
1669                 backlen = sldns_buffer_limit(c->buffer);
1670                 backup_p = memdup(sldns_buffer_begin(c->buffer), backlen);
1671                 if(!backup_p) {
1672                         log_err("malloc failure in serviced query callbacks");
1673                         error = NETEVENT_CLOSED;
1674                         c = NULL;
1675                 }
1676                 sq->outnet->svcd_overhead = backlen;
1677         }
1678         /* test the actual sq->cblist, because the next elem could be deleted*/
1679         while((p=sq->cblist) != NULL) {
1680                 sq->cblist = p->next; /* remove this element */
1681                 if(dobackup && c) {
1682                         sldns_buffer_clear(c->buffer);
1683                         sldns_buffer_write(c->buffer, backup_p, backlen);
1684                         sldns_buffer_flip(c->buffer);
1685                 }
1686                 fptr_ok(fptr_whitelist_serviced_query(p->cb));
1687                 (void)(*p->cb)(c, p->cb_arg, error, rep);
1688                 free(p);
1689         }
1690         if(backup_p) {
1691                 free(backup_p);
1692                 sq->outnet->svcd_overhead = 0;
1693         }
1694         verbose(VERB_ALGO, "svcd callbacks end");
1695         log_assert(sq->cblist == NULL);
1696         serviced_delete(sq);
1697 }
1698
1699 int
1700 serviced_tcp_callback(struct comm_point* c, void* arg, int error,
1701         struct comm_reply* rep)
1702 {
1703         struct serviced_query* sq = (struct serviced_query*)arg;
1704         struct comm_reply r2;
1705         sq->pending = NULL; /* removed after this callback */
1706         if(error != NETEVENT_NOERROR)
1707                 log_addr(VERB_QUERY, "tcp error for address",
1708                         &sq->addr, sq->addrlen);
1709         if(error==NETEVENT_NOERROR)
1710                 infra_update_tcp_works(sq->outnet->infra, &sq->addr,
1711                         sq->addrlen, sq->zone, sq->zonelen);
1712 #ifdef USE_DNSTAP
1713         if(error==NETEVENT_NOERROR && sq->outnet->dtenv &&
1714            (sq->outnet->dtenv->log_resolver_response_messages ||
1715             sq->outnet->dtenv->log_forwarder_response_messages))
1716                 dt_msg_send_outside_response(sq->outnet->dtenv, &sq->addr,
1717                 c->type, sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1718                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1719 #endif
1720         if(error==NETEVENT_NOERROR && sq->status == serviced_query_TCP_EDNS &&
1721                 (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1722                 LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(sldns_buffer_begin(
1723                 c->buffer)) == LDNS_RCODE_NOTIMPL) ) {
1724                 /* attempt to fallback to nonEDNS */
1725                 sq->status = serviced_query_TCP_EDNS_fallback;
1726                 serviced_tcp_initiate(sq, c->buffer);
1727                 return 0;
1728         } else if(error==NETEVENT_NOERROR &&
1729                 sq->status == serviced_query_TCP_EDNS_fallback &&
1730                         (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1731                         LDNS_RCODE_NOERROR || LDNS_RCODE_WIRE(
1732                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NXDOMAIN
1733                         || LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1734                         == LDNS_RCODE_YXDOMAIN)) {
1735                 /* the fallback produced a result that looks promising, note
1736                  * that this server should be approached without EDNS */
1737                 /* only store noEDNS in cache if domain is noDNSSEC */
1738                 if(!sq->want_dnssec)
1739                   if(!infra_edns_update(sq->outnet->infra, &sq->addr,
1740                         sq->addrlen, sq->zone, sq->zonelen, -1,
1741                         *sq->outnet->now_secs))
1742                         log_err("Out of memory caching no edns for host");
1743                 sq->status = serviced_query_TCP;
1744         }
1745         if(sq->tcp_upstream || sq->ssl_upstream) {
1746             struct timeval now = *sq->outnet->now_tv;
1747             if(now.tv_sec > sq->last_sent_time.tv_sec ||
1748                 (now.tv_sec == sq->last_sent_time.tv_sec &&
1749                 now.tv_usec > sq->last_sent_time.tv_usec)) {
1750                 /* convert from microseconds to milliseconds */
1751                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1752                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1753                 verbose(VERB_ALGO, "measured TCP-time at %d msec", roundtime);
1754                 log_assert(roundtime >= 0);
1755                 /* only store if less then AUTH_TIMEOUT seconds, it could be
1756                  * huge due to system-hibernated and we woke up */
1757                 if(roundtime < TCP_AUTH_QUERY_TIMEOUT*1000) {
1758                     if(!infra_rtt_update(sq->outnet->infra, &sq->addr,
1759                         sq->addrlen, sq->zone, sq->zonelen, sq->qtype,
1760                         roundtime, sq->last_rtt, (time_t)now.tv_sec))
1761                         log_err("out of memory noting rtt.");
1762                 }
1763             }
1764         }
1765         /* insert address into reply info */
1766         if(!rep) {
1767                 /* create one if there isn't (on errors) */
1768                 rep = &r2;
1769                 r2.c = c;
1770         }
1771         memcpy(&rep->addr, &sq->addr, sq->addrlen);
1772         rep->addrlen = sq->addrlen;
1773         serviced_callbacks(sq, error, c, rep);
1774         return 0;
1775 }
1776
1777 static void
1778 serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
1779 {
1780         verbose(VERB_ALGO, "initiate TCP query %s",
1781                 sq->status==serviced_query_TCP_EDNS?"EDNS":"");
1782         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1783         sq->last_sent_time = *sq->outnet->now_tv;
1784         sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1785                 serviced_tcp_callback, sq);
1786         if(!sq->pending) {
1787                 /* delete from tree so that a retry by above layer does not
1788                  * clash with this entry */
1789                 log_err("serviced_tcp_initiate: failed to send tcp query");
1790                 serviced_callbacks(sq, NETEVENT_CLOSED, NULL, NULL);
1791         }
1792 }
1793
1794 /** Send serviced query over TCP return false on initial failure */
1795 static int
1796 serviced_tcp_send(struct serviced_query* sq, sldns_buffer* buff)
1797 {
1798         int vs, rtt;
1799         uint8_t edns_lame_known;
1800         if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, sq->zone,
1801                 sq->zonelen, *sq->outnet->now_secs, &vs, &edns_lame_known,
1802                 &rtt))
1803                 return 0;
1804         if(vs != -1)
1805                 sq->status = serviced_query_TCP_EDNS;
1806         else    sq->status = serviced_query_TCP;
1807         serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
1808         sq->last_sent_time = *sq->outnet->now_tv;
1809         sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
1810                 serviced_tcp_callback, sq);
1811         return sq->pending != NULL;
1812 }
1813
1814 /* see if packet is edns malformed; got zeroes at start.
1815  * This is from servers that return malformed packets to EDNS0 queries,
1816  * but they return good packets for nonEDNS0 queries.
1817  * We try to detect their output; without resorting to a full parse or
1818  * check for too many bytes after the end of the packet. */
1819 static int
1820 packet_edns_malformed(struct sldns_buffer* buf, int qtype)
1821 {
1822         size_t len;
1823         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE)
1824                 return 1; /* malformed */
1825         /* they have NOERROR rcode, 1 answer. */
1826         if(LDNS_RCODE_WIRE(sldns_buffer_begin(buf)) != LDNS_RCODE_NOERROR)
1827                 return 0;
1828         /* one query (to skip) and answer records */
1829         if(LDNS_QDCOUNT(sldns_buffer_begin(buf)) != 1 ||
1830                 LDNS_ANCOUNT(sldns_buffer_begin(buf)) == 0)
1831                 return 0;
1832         /* skip qname */
1833         len = dname_valid(sldns_buffer_at(buf, LDNS_HEADER_SIZE),
1834                 sldns_buffer_limit(buf)-LDNS_HEADER_SIZE);
1835         if(len == 0)
1836                 return 0;
1837         if(len == 1 && qtype == 0)
1838                 return 0; /* we asked for '.' and type 0 */
1839         /* and then 4 bytes (type and class of query) */
1840         if(sldns_buffer_limit(buf) < LDNS_HEADER_SIZE + len + 4 + 3)
1841                 return 0;
1842
1843         /* and start with 11 zeroes as the answer RR */
1844         /* so check the qtype of the answer record, qname=0, type=0 */
1845         if(sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[0] == 0 &&
1846            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[1] == 0 &&
1847            sldns_buffer_at(buf, LDNS_HEADER_SIZE+len+4)[2] == 0)
1848                 return 1;
1849         return 0;
1850 }
1851
1852 int
1853 serviced_udp_callback(struct comm_point* c, void* arg, int error,
1854         struct comm_reply* rep)
1855 {
1856         struct serviced_query* sq = (struct serviced_query*)arg;
1857         struct outside_network* outnet = sq->outnet;
1858         struct timeval now = *sq->outnet->now_tv;
1859         int fallback_tcp = 0;
1860
1861         sq->pending = NULL; /* removed after callback */
1862         if(error == NETEVENT_TIMEOUT) {
1863                 int rto = 0;
1864                 if(sq->status == serviced_query_PROBE_EDNS) {
1865                         /* non-EDNS probe failed; we do not know its status,
1866                          * keep trying with EDNS, timeout may not be caused
1867                          * by EDNS. */
1868                         sq->status = serviced_query_UDP_EDNS;
1869                 }
1870                 if(sq->status == serviced_query_UDP_EDNS && sq->last_rtt < 5000) {
1871                         /* fallback to 1480/1280 */
1872                         sq->status = serviced_query_UDP_EDNS_FRAG;
1873                         log_name_addr(VERB_ALGO, "try edns1xx0", sq->qbuf+10,
1874                                 &sq->addr, sq->addrlen);
1875                         if(!serviced_udp_send(sq, c->buffer)) {
1876                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1877                         }
1878                         return 0;
1879                 }
1880                 if(sq->status == serviced_query_UDP_EDNS_FRAG) {
1881                         /* fragmentation size did not fix it */
1882                         sq->status = serviced_query_UDP_EDNS;
1883                 }
1884                 sq->retry++;
1885                 if(!(rto=infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1886                         sq->zone, sq->zonelen, sq->qtype, -1, sq->last_rtt,
1887                         (time_t)now.tv_sec)))
1888                         log_err("out of memory in UDP exponential backoff");
1889                 if(sq->retry < OUTBOUND_UDP_RETRY) {
1890                         log_name_addr(VERB_ALGO, "retry query", sq->qbuf+10,
1891                                 &sq->addr, sq->addrlen);
1892                         if(!serviced_udp_send(sq, c->buffer)) {
1893                                 serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1894                         }
1895                         return 0;
1896                 }
1897                 if(rto >= RTT_MAX_TIMEOUT) {
1898                         fallback_tcp = 1;
1899                         /* UDP does not work, fallback to TCP below */
1900                 } else {
1901                         serviced_callbacks(sq, NETEVENT_TIMEOUT, c, rep);
1902                         return 0;
1903                 }
1904         } else if(error != NETEVENT_NOERROR) {
1905                 /* udp returns error (due to no ID or interface available) */
1906                 serviced_callbacks(sq, error, c, rep);
1907                 return 0;
1908         }
1909 #ifdef USE_DNSTAP
1910         if(error == NETEVENT_NOERROR && outnet->dtenv &&
1911            (outnet->dtenv->log_resolver_response_messages ||
1912             outnet->dtenv->log_forwarder_response_messages))
1913                 dt_msg_send_outside_response(outnet->dtenv, &sq->addr, c->type,
1914                 sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
1915                 &sq->last_sent_time, sq->outnet->now_tv, c->buffer);
1916 #endif
1917         if(!fallback_tcp) {
1918             if( (sq->status == serviced_query_UDP_EDNS
1919                 ||sq->status == serviced_query_UDP_EDNS_FRAG)
1920                 && (LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer))
1921                         == LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(
1922                         sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOTIMPL
1923                     || packet_edns_malformed(c->buffer, sq->qtype)
1924                         )) {
1925                 /* try to get an answer by falling back without EDNS */
1926                 verbose(VERB_ALGO, "serviced query: attempt without EDNS");
1927                 sq->status = serviced_query_UDP_EDNS_fallback;
1928                 sq->retry = 0;
1929                 if(!serviced_udp_send(sq, c->buffer)) {
1930                         serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
1931                 }
1932                 return 0;
1933             } else if(sq->status == serviced_query_PROBE_EDNS) {
1934                 /* probe without EDNS succeeds, so we conclude that this
1935                  * host likely has EDNS packets dropped */
1936                 log_addr(VERB_DETAIL, "timeouts, concluded that connection to "
1937                         "host drops EDNS packets", &sq->addr, sq->addrlen);
1938                 /* only store noEDNS in cache if domain is noDNSSEC */
1939                 if(!sq->want_dnssec)
1940                   if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1941                         sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
1942                         log_err("Out of memory caching no edns for host");
1943                   }
1944                 sq->status = serviced_query_UDP;
1945             } else if(sq->status == serviced_query_UDP_EDNS &&
1946                 !sq->edns_lame_known) {
1947                 /* now we know that edns queries received answers store that */
1948                 log_addr(VERB_ALGO, "serviced query: EDNS works for",
1949                         &sq->addr, sq->addrlen);
1950                 if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1951                         sq->zone, sq->zonelen, 0, (time_t)now.tv_sec)) {
1952                         log_err("Out of memory caching edns works");
1953                 }
1954                 sq->edns_lame_known = 1;
1955             } else if(sq->status == serviced_query_UDP_EDNS_fallback &&
1956                 !sq->edns_lame_known && (LDNS_RCODE_WIRE(
1957                 sldns_buffer_begin(c->buffer)) == LDNS_RCODE_NOERROR ||
1958                 LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) ==
1959                 LDNS_RCODE_NXDOMAIN || LDNS_RCODE_WIRE(sldns_buffer_begin(
1960                 c->buffer)) == LDNS_RCODE_YXDOMAIN)) {
1961                 /* the fallback produced a result that looks promising, note
1962                  * that this server should be approached without EDNS */
1963                 /* only store noEDNS in cache if domain is noDNSSEC */
1964                 if(!sq->want_dnssec) {
1965                   log_addr(VERB_ALGO, "serviced query: EDNS fails for",
1966                         &sq->addr, sq->addrlen);
1967                   if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
1968                         sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
1969                         log_err("Out of memory caching no edns for host");
1970                   }
1971                 } else {
1972                   log_addr(VERB_ALGO, "serviced query: EDNS fails, but "
1973                         "not stored because need DNSSEC for", &sq->addr,
1974                         sq->addrlen);
1975                 }
1976                 sq->status = serviced_query_UDP;
1977             }
1978             if(now.tv_sec > sq->last_sent_time.tv_sec ||
1979                 (now.tv_sec == sq->last_sent_time.tv_sec &&
1980                 now.tv_usec > sq->last_sent_time.tv_usec)) {
1981                 /* convert from microseconds to milliseconds */
1982                 int roundtime = ((int)(now.tv_sec - sq->last_sent_time.tv_sec))*1000
1983                   + ((int)now.tv_usec - (int)sq->last_sent_time.tv_usec)/1000;
1984                 verbose(VERB_ALGO, "measured roundtrip at %d msec", roundtime);
1985                 log_assert(roundtime >= 0);
1986                 /* in case the system hibernated, do not enter a huge value,
1987                  * above this value gives trouble with server selection */
1988                 if(roundtime < 60000) {
1989                     if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
1990                         sq->zone, sq->zonelen, sq->qtype, roundtime,
1991                         sq->last_rtt, (time_t)now.tv_sec))
1992                         log_err("out of memory noting rtt.");
1993                 }
1994             }
1995         } /* end of if_!fallback_tcp */
1996         /* perform TC flag check and TCP fallback after updating our
1997          * cache entries for EDNS status and RTT times */
1998         if(LDNS_TC_WIRE(sldns_buffer_begin(c->buffer)) || fallback_tcp) {
1999                 /* fallback to TCP */
2000                 /* this discards partial UDP contents */
2001                 if(sq->status == serviced_query_UDP_EDNS ||
2002                         sq->status == serviced_query_UDP_EDNS_FRAG ||
2003                         sq->status == serviced_query_UDP_EDNS_fallback)
2004                         /* if we have unfinished EDNS_fallback, start again */
2005                         sq->status = serviced_query_TCP_EDNS;
2006                 else    sq->status = serviced_query_TCP;
2007                 serviced_tcp_initiate(sq, c->buffer);
2008                 return 0;
2009         }
2010         /* yay! an answer */
2011         serviced_callbacks(sq, error, c, rep);
2012         return 0;
2013 }
2014
2015 struct serviced_query*
2016 outnet_serviced_query(struct outside_network* outnet,
2017         struct query_info* qinfo, uint16_t flags, int dnssec, int want_dnssec,
2018         int nocaps, int tcp_upstream, int ssl_upstream,
2019         struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
2020         size_t zonelen, struct module_qstate* qstate,
2021         comm_point_callback_type* callback, void* callback_arg, sldns_buffer* buff,
2022         struct module_env* env)
2023 {
2024         struct serviced_query* sq;
2025         struct service_callback* cb;
2026         if(!inplace_cb_query_call(env, qinfo, flags, addr, addrlen, zone, zonelen,
2027                 qstate, qstate->region))
2028                         return NULL;
2029         serviced_gen_query(buff, qinfo->qname, qinfo->qname_len, qinfo->qtype,
2030                 qinfo->qclass, flags);
2031         sq = lookup_serviced(outnet, buff, dnssec, addr, addrlen,
2032                 qstate->edns_opts_back_out);
2033         /* duplicate entries are included in the callback list, because
2034          * there is a counterpart registration by our caller that needs to
2035          * be doubly-removed (with callbacks perhaps). */
2036         if(!(cb = (struct service_callback*)malloc(sizeof(*cb))))
2037                 return NULL;
2038         if(!sq) {
2039                 /* make new serviced query entry */
2040                 sq = serviced_create(outnet, buff, dnssec, want_dnssec, nocaps,
2041                         tcp_upstream, ssl_upstream, addr, addrlen, zone,
2042                         zonelen, (int)qinfo->qtype, qstate->edns_opts_back_out);
2043                 if(!sq) {
2044                         free(cb);
2045                         return NULL;
2046                 }
2047                 /* perform first network action */
2048                 if(outnet->do_udp && !(tcp_upstream || ssl_upstream)) {
2049                         if(!serviced_udp_send(sq, buff)) {
2050                                 (void)rbtree_delete(outnet->serviced, sq);
2051                                 free(sq->qbuf);
2052                                 free(sq->zone);
2053                                 free(sq);
2054                                 free(cb);
2055                                 return NULL;
2056                         }
2057                 } else {
2058                         if(!serviced_tcp_send(sq, buff)) {
2059                                 (void)rbtree_delete(outnet->serviced, sq);
2060                                 free(sq->qbuf);
2061                                 free(sq->zone);
2062                                 free(sq);
2063                                 free(cb);
2064                                 return NULL;
2065                         }
2066                 }
2067         }
2068         /* add callback to list of callbacks */
2069         cb->cb = callback;
2070         cb->cb_arg = callback_arg;
2071         cb->next = sq->cblist;
2072         sq->cblist = cb;
2073         return sq;
2074 }
2075
2076 /** remove callback from list */
2077 static void
2078 callback_list_remove(struct serviced_query* sq, void* cb_arg)
2079 {
2080         struct service_callback** pp = &sq->cblist;
2081         while(*pp) {
2082                 if((*pp)->cb_arg == cb_arg) {
2083                         struct service_callback* del = *pp;
2084                         *pp = del->next;
2085                         free(del);
2086                         return;
2087                 }
2088                 pp = &(*pp)->next;
2089         }
2090 }
2091
2092 void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
2093 {
2094         if(!sq)
2095                 return;
2096         callback_list_remove(sq, cb_arg);
2097         /* if callbacks() routine scheduled deletion, let it do that */
2098         if(!sq->cblist && !sq->to_be_deleted) {
2099                 (void)rbtree_delete(sq->outnet->serviced, sq);
2100                 serviced_delete(sq);
2101         }
2102 }
2103
2104 /** get memory used by waiting tcp entry (in use or not) */
2105 static size_t
2106 waiting_tcp_get_mem(struct waiting_tcp* w)
2107 {
2108         size_t s;
2109         if(!w) return 0;
2110         s = sizeof(*w) + w->pkt_len;
2111         if(w->timer)
2112                 s += comm_timer_get_mem(w->timer);
2113         return s;
2114 }
2115
2116 /** get memory used by port if */
2117 static size_t
2118 if_get_mem(struct port_if* pif)
2119 {
2120         size_t s;
2121         int i;
2122         s = sizeof(*pif) + sizeof(int)*pif->avail_total +
2123                 sizeof(struct port_comm*)*pif->maxout;
2124         for(i=0; i<pif->inuse; i++)
2125                 s += sizeof(*pif->out[i]) +
2126                         comm_point_get_mem(pif->out[i]->cp);
2127         return s;
2128 }
2129
2130 /** get memory used by waiting udp */
2131 static size_t
2132 waiting_udp_get_mem(struct pending* w)
2133 {
2134         size_t s;
2135         s = sizeof(*w) + comm_timer_get_mem(w->timer) + w->pkt_len;
2136         return s;
2137 }
2138
2139 size_t outnet_get_mem(struct outside_network* outnet)
2140 {
2141         size_t i;
2142         int k;
2143         struct waiting_tcp* w;
2144         struct pending* u;
2145         struct serviced_query* sq;
2146         struct service_callback* sb;
2147         struct port_comm* pc;
2148         size_t s = sizeof(*outnet) + sizeof(*outnet->base) +
2149                 sizeof(*outnet->udp_buff) +
2150                 sldns_buffer_capacity(outnet->udp_buff);
2151         /* second buffer is not ours */
2152         for(pc = outnet->unused_fds; pc; pc = pc->next) {
2153                 s += sizeof(*pc) + comm_point_get_mem(pc->cp);
2154         }
2155         for(k=0; k<outnet->num_ip4; k++)
2156                 s += if_get_mem(&outnet->ip4_ifs[k]);
2157         for(k=0; k<outnet->num_ip6; k++)
2158                 s += if_get_mem(&outnet->ip6_ifs[k]);
2159         for(u=outnet->udp_wait_first; u; u=u->next_waiting)
2160                 s += waiting_udp_get_mem(u);
2161
2162         s += sizeof(struct pending_tcp*)*outnet->num_tcp;
2163         for(i=0; i<outnet->num_tcp; i++) {
2164                 s += sizeof(struct pending_tcp);
2165                 s += comm_point_get_mem(outnet->tcp_conns[i]->c);
2166                 if(outnet->tcp_conns[i]->query)
2167                         s += waiting_tcp_get_mem(outnet->tcp_conns[i]->query);
2168         }
2169         for(w=outnet->tcp_wait_first; w; w = w->next_waiting)
2170                 s += waiting_tcp_get_mem(w);
2171         s += sizeof(*outnet->pending);
2172         s += (sizeof(struct pending) + comm_timer_get_mem(NULL)) *
2173                 outnet->pending->count;
2174         s += sizeof(*outnet->serviced);
2175         s += outnet->svcd_overhead;
2176         RBTREE_FOR(sq, struct serviced_query*, outnet->serviced) {
2177                 s += sizeof(*sq) + sq->qbuflen;
2178                 for(sb = sq->cblist; sb; sb = sb->next)
2179                         s += sizeof(*sb);
2180         }
2181         return s;
2182 }
2183
2184 size_t
2185 serviced_get_mem(struct serviced_query* sq)
2186 {
2187         struct service_callback* sb;
2188         size_t s;
2189         s = sizeof(*sq) + sq->qbuflen;
2190         for(sb = sq->cblist; sb; sb = sb->next)
2191                 s += sizeof(*sb);
2192         if(sq->status == serviced_query_UDP_EDNS ||
2193                 sq->status == serviced_query_UDP ||
2194                 sq->status == serviced_query_PROBE_EDNS ||
2195                 sq->status == serviced_query_UDP_EDNS_FRAG ||
2196                 sq->status == serviced_query_UDP_EDNS_fallback) {
2197                 s += sizeof(struct pending);
2198                 s += comm_timer_get_mem(NULL);
2199         } else {
2200                 /* does not have size of the pkt pointer */
2201                 /* always has a timer except on malloc failures */
2202
2203                 /* these sizes are part of the main outside network mem */
2204                 /*
2205                 s += sizeof(struct waiting_tcp);
2206                 s += comm_timer_get_mem(NULL);
2207                 */
2208         }
2209         return s;
2210 }
2211