contrib/unbound/services/authzone.c

   1 /*
   2  * services/authzone.c - authoritative zone that is locally hosted.
   3  *
   4  * Copyright (c) 2017, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains the functions for an authority zone.  This zone
  40  * is queried by the iterator, just like a stub or forward zone, but then
  41  * the data is locally held.
  42  */
  43
  44 #include "config.h"
  45 #include "services/authzone.h"
  46 #include "util/data/dname.h"
  47 #include "util/data/msgparse.h"
  48 #include "util/data/msgreply.h"
  49 #include "util/data/msgencode.h"
  50 #include "util/data/packed_rrset.h"
  51 #include "util/regional.h"
  52 #include "util/net_help.h"
  53 #include "util/netevent.h"
  54 #include "util/config_file.h"
  55 #include "util/log.h"
  56 #include "util/module.h"
  57 #include "util/random.h"
  58 #include "services/cache/dns.h"
  59 #include "services/outside_network.h"
  60 #include "services/listen_dnsport.h"
  61 #include "services/mesh.h"
  62 #include "sldns/rrdef.h"
  63 #include "sldns/pkthdr.h"
  64 #include "sldns/sbuffer.h"
  65 #include "sldns/str2wire.h"
  66 #include "sldns/wire2str.h"
  67 #include "sldns/parseutil.h"
  68 #include "sldns/keyraw.h"
  69 #include "validator/val_nsec3.h"
  70 #include "validator/val_secalgo.h"
  71 #include <ctype.h>
  72
  73 /** bytes to use for NSEC3 hash buffer. 20 for sha1 */
  74 #define N3HASHBUFLEN 32
  75 /** max number of CNAMEs we are willing to follow (in one answer) */
  76 #define MAX_CNAME_CHAIN 8
  77 /** timeout for probe packets for SOA */
  78 #define AUTH_PROBE_TIMEOUT 100 /* msec */
  79 /** when to stop with SOA probes (when exponential timeouts exceed this) */
  80 #define AUTH_PROBE_TIMEOUT_STOP 1000 /* msec */
  81 /* auth transfer timeout for TCP connections, in msec */
  82 #define AUTH_TRANSFER_TIMEOUT 10000 /* msec */
  83 /* auth transfer max backoff for failed tranfers and probes */
  84 #define AUTH_TRANSFER_MAX_BACKOFF 86400 /* sec */
  85 /* auth http port number */
  86 #define AUTH_HTTP_PORT 80
  87 /* auth https port number */
  88 #define AUTH_HTTPS_PORT 443
  89 /* max depth for nested $INCLUDEs */
  90 #define MAX_INCLUDE_DEPTH 10
  91
  92 /** pick up nextprobe task to start waiting to perform transfer actions */
  93 static void xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
  94         int failure, int lookup_only);
  95 /** move to sending the probe packets, next if fails. task_probe */
  96 static void xfr_probe_send_or_end(struct auth_xfer* xfr,
  97         struct module_env* env);
  98 /** pick up probe task with specified(or NULL) destination first,
  99  * or transfer task if nothing to probe, or false if already in progress */
 100 static int xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
 101         struct auth_master* spec);
 102 /** delete xfer structure (not its tree entry) */
 103 static void auth_xfer_delete(struct auth_xfer* xfr);
 104
 105 /** create new dns_msg */
 106 static struct dns_msg*
 107 msg_create(struct regional* region, struct query_info* qinfo)
 108 {
 109         struct dns_msg* msg = (struct dns_msg*)regional_alloc(region,
 110                 sizeof(struct dns_msg));
 111         if(!msg)
 112                 return NULL;
 113         msg->qinfo.qname = regional_alloc_init(region, qinfo->qname,
 114                 qinfo->qname_len);
 115         if(!msg->qinfo.qname)
 116                 return NULL;
 117         msg->qinfo.qname_len = qinfo->qname_len;
 118         msg->qinfo.qtype = qinfo->qtype;
 119         msg->qinfo.qclass = qinfo->qclass;
 120         msg->qinfo.local_alias = NULL;
 121         /* non-packed reply_info, because it needs to grow the array */
 122         msg->rep = (struct reply_info*)regional_alloc_zero(region,
 123                 sizeof(struct reply_info)-sizeof(struct rrset_ref));
 124         if(!msg->rep)
 125                 return NULL;
 126         msg->rep->flags = (uint16_t)(BIT_QR | BIT_AA);
 127         msg->rep->authoritative = 1;
 128         msg->rep->qdcount = 1;
 129         /* rrsets is NULL, no rrsets yet */
 130         return msg;
 131 }
 132
 133 /** grow rrset array by one in msg */
 134 static int
 135 msg_grow_array(struct regional* region, struct dns_msg* msg)
 136 {
 137         if(msg->rep->rrsets == NULL) {
 138                 msg->rep->rrsets = regional_alloc_zero(region,
 139                         sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
 140                 if(!msg->rep->rrsets)
 141                         return 0;
 142         } else {
 143                 struct ub_packed_rrset_key** rrsets_old = msg->rep->rrsets;
 144                 msg->rep->rrsets = regional_alloc_zero(region,
 145                         sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
 146                 if(!msg->rep->rrsets)
 147                         return 0;
 148                 memmove(msg->rep->rrsets, rrsets_old,
 149                         sizeof(struct ub_packed_rrset_key*)*msg->rep->rrset_count);
 150         }
 151         return 1;
 152 }
 153
 154 /** get ttl of rrset */
 155 static time_t
 156 get_rrset_ttl(struct ub_packed_rrset_key* k)
 157 {
 158         struct packed_rrset_data* d = (struct packed_rrset_data*)
 159                 k->entry.data;
 160         return d->ttl;
 161 }
 162
 163 /** Copy rrset into region from domain-datanode and packet rrset */
 164 static struct ub_packed_rrset_key*
 165 auth_packed_rrset_copy_region(struct auth_zone* z, struct auth_data* node,
 166         struct auth_rrset* rrset, struct regional* region, time_t adjust)
 167 {
 168         struct ub_packed_rrset_key key;
 169         memset(&key, 0, sizeof(key));
 170         key.entry.key = &key;
 171         key.entry.data = rrset->data;
 172         key.rk.dname = node->name;
 173         key.rk.dname_len = node->namelen;
 174         key.rk.type = htons(rrset->type);
 175         key.rk.rrset_class = htons(z->dclass);
 176         key.entry.hash = rrset_key_hash(&key.rk);
 177         return packed_rrset_copy_region(&key, region, adjust);
 178 }
 179
 180 /** fix up msg->rep TTL and prefetch ttl */
 181 static void
 182 msg_ttl(struct dns_msg* msg)
 183 {
 184         if(msg->rep->rrset_count == 0) return;
 185         if(msg->rep->rrset_count == 1) {
 186                 msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
 187                 msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 188                 msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
 189         } else if(get_rrset_ttl(msg->rep->rrsets[msg->rep->rrset_count-1]) <
 190                 msg->rep->ttl) {
 191                 msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[
 192                         msg->rep->rrset_count-1]);
 193                 msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 194                 msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
 195         }
 196 }
 197
 198 /** see if rrset is a duplicate in the answer message */
 199 static int
 200 msg_rrset_duplicate(struct dns_msg* msg, uint8_t* nm, size_t nmlen,
 201         uint16_t type, uint16_t dclass)
 202 {
 203         size_t i;
 204         for(i=0; i<msg->rep->rrset_count; i++) {
 205                 struct ub_packed_rrset_key* k = msg->rep->rrsets[i];
 206                 if(ntohs(k->rk.type) == type && k->rk.dname_len == nmlen &&
 207                         ntohs(k->rk.rrset_class) == dclass &&
 208                         query_dname_compare(k->rk.dname, nm) == 0)
 209                         return 1;
 210         }
 211         return 0;
 212 }
 213
 214 /** add rrset to answer section (no auth, add rrsets yet) */
 215 static int
 216 msg_add_rrset_an(struct auth_zone* z, struct regional* region,
 217         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 218 {
 219         log_assert(msg->rep->ns_numrrsets == 0);
 220         log_assert(msg->rep->ar_numrrsets == 0);
 221         if(!rrset)
 222                 return 1;
 223         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 224                 z->dclass))
 225                 return 1;
 226         /* grow array */
 227         if(!msg_grow_array(region, msg))
 228                 return 0;
 229         /* copy it */
 230         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 231                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 232                 return 0;
 233         msg->rep->rrset_count++;
 234         msg->rep->an_numrrsets++;
 235         msg_ttl(msg);
 236         return 1;
 237 }
 238
 239 /** add rrset to authority section (no additonal section rrsets yet) */
 240 static int
 241 msg_add_rrset_ns(struct auth_zone* z, struct regional* region,
 242         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 243 {
 244         log_assert(msg->rep->ar_numrrsets == 0);
 245         if(!rrset)
 246                 return 1;
 247         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 248                 z->dclass))
 249                 return 1;
 250         /* grow array */
 251         if(!msg_grow_array(region, msg))
 252                 return 0;
 253         /* copy it */
 254         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 255                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 256                 return 0;
 257         msg->rep->rrset_count++;
 258         msg->rep->ns_numrrsets++;
 259         msg_ttl(msg);
 260         return 1;
 261 }
 262
 263 /** add rrset to additional section */
 264 static int
 265 msg_add_rrset_ar(struct auth_zone* z, struct regional* region,
 266         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 267 {
 268         if(!rrset)
 269                 return 1;
 270         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 271                 z->dclass))
 272                 return 1;
 273         /* grow array */
 274         if(!msg_grow_array(region, msg))
 275                 return 0;
 276         /* copy it */
 277         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 278                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 279                 return 0;
 280         msg->rep->rrset_count++;
 281         msg->rep->ar_numrrsets++;
 282         msg_ttl(msg);
 283         return 1;
 284 }
 285
 286 struct auth_zones* auth_zones_create(void)
 287 {
 288         struct auth_zones* az = (struct auth_zones*)calloc(1, sizeof(*az));
 289         if(!az) {
 290                 log_err("out of memory");
 291                 return NULL;
 292         }
 293         rbtree_init(&az->ztree, &auth_zone_cmp);
 294         rbtree_init(&az->xtree, &auth_xfer_cmp);
 295         lock_rw_init(&az->lock);
 296         lock_protect(&az->lock, &az->ztree, sizeof(az->ztree));
 297         lock_protect(&az->lock, &az->xtree, sizeof(az->xtree));
 298         /* also lock protects the rbnode's in struct auth_zone, auth_xfer */
 299         return az;
 300 }
 301
 302 int auth_zone_cmp(const void* z1, const void* z2)
 303 {
 304         /* first sort on class, so that hierarchy can be maintained within
 305          * a class */
 306         struct auth_zone* a = (struct auth_zone*)z1;
 307         struct auth_zone* b = (struct auth_zone*)z2;
 308         int m;
 309         if(a->dclass != b->dclass) {
 310                 if(a->dclass < b->dclass)
 311                         return -1;
 312                 return 1;
 313         }
 314         /* sorted such that higher zones sort before lower zones (their
 315          * contents) */
 316         return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
 317 }
 318
 319 int auth_data_cmp(const void* z1, const void* z2)
 320 {
 321         struct auth_data* a = (struct auth_data*)z1;
 322         struct auth_data* b = (struct auth_data*)z2;
 323         int m;
 324         /* canonical sort, because DNSSEC needs that */
 325         return dname_canon_lab_cmp(a->name, a->namelabs, b->name,
 326                 b->namelabs, &m);
 327 }
 328
 329 int auth_xfer_cmp(const void* z1, const void* z2)
 330 {
 331         /* first sort on class, so that hierarchy can be maintained within
 332          * a class */
 333         struct auth_xfer* a = (struct auth_xfer*)z1;
 334         struct auth_xfer* b = (struct auth_xfer*)z2;
 335         int m;
 336         if(a->dclass != b->dclass) {
 337                 if(a->dclass < b->dclass)
 338                         return -1;
 339                 return 1;
 340         }
 341         /* sorted such that higher zones sort before lower zones (their
 342          * contents) */
 343         return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
 344 }
 345
 346 /** delete auth rrset node */
 347 static void
 348 auth_rrset_delete(struct auth_rrset* rrset)
 349 {
 350         if(!rrset) return;
 351         free(rrset->data);
 352         free(rrset);
 353 }
 354
 355 /** delete auth data domain node */
 356 static void
 357 auth_data_delete(struct auth_data* n)
 358 {
 359         struct auth_rrset* p, *np;
 360         if(!n) return;
 361         p = n->rrsets;
 362         while(p) {
 363                 np = p->next;
 364                 auth_rrset_delete(p);
 365                 p = np;
 366         }
 367         free(n->name);
 368         free(n);
 369 }
 370
 371 /** helper traverse to delete zones */
 372 static void
 373 auth_data_del(rbnode_type* n, void* ATTR_UNUSED(arg))
 374 {
 375         struct auth_data* z = (struct auth_data*)n->key;
 376         auth_data_delete(z);
 377 }
 378
 379 /** delete an auth zone structure (tree remove must be done elsewhere) */
 380 static void
 381 auth_zone_delete(struct auth_zone* z)
 382 {
 383         if(!z) return;
 384         lock_rw_destroy(&z->lock);
 385         traverse_postorder(&z->data, auth_data_del, NULL);
 386         free(z->name);
 387         free(z->zonefile);
 388         free(z);
 389 }
 390
 391 struct auth_zone*
 392 auth_zone_create(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 393         uint16_t dclass)
 394 {
 395         struct auth_zone* z = (struct auth_zone*)calloc(1, sizeof(*z));
 396         if(!z) {
 397                 return NULL;
 398         }
 399         z->node.key = z;
 400         z->dclass = dclass;
 401         z->namelen = nmlen;
 402         z->namelabs = dname_count_labels(nm);
 403         z->name = memdup(nm, nmlen);
 404         if(!z->name) {
 405                 free(z);
 406                 return NULL;
 407         }
 408         rbtree_init(&z->data, &auth_data_cmp);
 409         lock_rw_init(&z->lock);
 410         lock_protect(&z->lock, &z->name, sizeof(*z)-sizeof(rbnode_type));
 411         lock_rw_wrlock(&z->lock);
 412         /* z lock protects all, except rbtree itself, which is az->lock */
 413         if(!rbtree_insert(&az->ztree, &z->node)) {
 414                 lock_rw_unlock(&z->lock);
 415                 auth_zone_delete(z);
 416                 log_warn("duplicate auth zone");
 417                 return NULL;
 418         }
 419         return z;
 420 }
 421
 422 struct auth_zone*
 423 auth_zone_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 424         uint16_t dclass)
 425 {
 426         struct auth_zone key;
 427         key.node.key = &key;
 428         key.dclass = dclass;
 429         key.name = nm;
 430         key.namelen = nmlen;
 431         key.namelabs = dname_count_labels(nm);
 432         return (struct auth_zone*)rbtree_search(&az->ztree, &key);
 433 }
 434
 435 struct auth_xfer*
 436 auth_xfer_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 437         uint16_t dclass)
 438 {
 439         struct auth_xfer key;
 440         key.node.key = &key;
 441         key.dclass = dclass;
 442         key.name = nm;
 443         key.namelen = nmlen;
 444         key.namelabs = dname_count_labels(nm);
 445         return (struct auth_xfer*)rbtree_search(&az->xtree, &key);
 446 }
 447
 448 /** find an auth zone or sorted less-or-equal, return true if exact */
 449 static int
 450 auth_zone_find_less_equal(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 451         uint16_t dclass, struct auth_zone** z)
 452 {
 453         struct auth_zone key;
 454         key.node.key = &key;
 455         key.dclass = dclass;
 456         key.name = nm;
 457         key.namelen = nmlen;
 458         key.namelabs = dname_count_labels(nm);
 459         return rbtree_find_less_equal(&az->ztree, &key, (rbnode_type**)z);
 460 }
 461
 462
 463 /** find the auth zone that is above the given name */
 464 struct auth_zone*
 465 auth_zones_find_zone(struct auth_zones* az, uint8_t* name, size_t name_len,
 466         uint16_t dclass)
 467 {
 468         uint8_t* nm = name;
 469         size_t nmlen = name_len;
 470         struct auth_zone* z;
 471         if(auth_zone_find_less_equal(az, nm, nmlen, dclass, &z)) {
 472                 /* exact match */
 473                 return z;
 474         } else {
 475                 /* less-or-nothing */
 476                 if(!z) return NULL; /* nothing smaller, nothing above it */
 477                 /* we found smaller name; smaller may be above the name,
 478                  * but not below it. */
 479                 nm = dname_get_shared_topdomain(z->name, name);
 480                 dname_count_size_labels(nm, &nmlen);
 481                 z = NULL;
 482         }
 483
 484         /* search up */
 485         while(!z) {
 486                 z = auth_zone_find(az, nm, nmlen, dclass);
 487                 if(z) return z;
 488                 if(dname_is_root(nm)) break;
 489                 dname_remove_label(&nm, &nmlen);
 490         }
 491         return NULL;
 492 }
 493
 494 /** find or create zone with name str. caller must have lock on az.
 495  * returns a wrlocked zone */
 496 static struct auth_zone*
 497 auth_zones_find_or_add_zone(struct auth_zones* az, char* name)
 498 {
 499         uint8_t nm[LDNS_MAX_DOMAINLEN+1];
 500         size_t nmlen = sizeof(nm);
 501         struct auth_zone* z;
 502
 503         if(sldns_str2wire_dname_buf(name, nm, &nmlen) != 0) {
 504                 log_err("cannot parse auth zone name: %s", name);
 505                 return 0;
 506         }
 507         z = auth_zone_find(az, nm, nmlen, LDNS_RR_CLASS_IN);
 508         if(!z) {
 509                 /* not found, create the zone */
 510                 z = auth_zone_create(az, nm, nmlen, LDNS_RR_CLASS_IN);
 511         } else {
 512                 lock_rw_wrlock(&z->lock);
 513         }
 514         return z;
 515 }
 516
 517 /** find or create xfer zone with name str. caller must have lock on az.
 518  * returns a locked xfer */
 519 static struct auth_xfer*
 520 auth_zones_find_or_add_xfer(struct auth_zones* az, struct auth_zone* z)
 521 {
 522         struct auth_xfer* x;
 523         x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
 524         if(!x) {
 525                 /* not found, create the zone */
 526                 x = auth_xfer_create(az, z);
 527         } else {
 528                 lock_basic_lock(&x->lock);
 529         }
 530         return x;
 531 }
 532
 533 int
 534 auth_zone_set_zonefile(struct auth_zone* z, char* zonefile)
 535 {
 536         if(z->zonefile) free(z->zonefile);
 537         if(zonefile == NULL) {
 538                 z->zonefile = NULL;
 539         } else {
 540                 z->zonefile = strdup(zonefile);
 541                 if(!z->zonefile) {
 542                         log_err("malloc failure");
 543                         return 0;
 544                 }
 545         }
 546         return 1;
 547 }
 548
 549 /** set auth zone fallback. caller must have lock on zone */
 550 int
 551 auth_zone_set_fallback(struct auth_zone* z, char* fallbackstr)
 552 {
 553         if(strcmp(fallbackstr, "yes") != 0 && strcmp(fallbackstr, "no") != 0){
 554                 log_err("auth zone fallback, expected yes or no, got %s",
 555                         fallbackstr);
 556                 return 0;
 557         }
 558         z->fallback_enabled = (strcmp(fallbackstr, "yes")==0);
 559         return 1;
 560 }
 561
 562 /** create domain with the given name */
 563 static struct auth_data*
 564 az_domain_create(struct auth_zone* z, uint8_t* nm, size_t nmlen)
 565 {
 566         struct auth_data* n = (struct auth_data*)malloc(sizeof(*n));
 567         if(!n) return NULL;
 568         memset(n, 0, sizeof(*n));
 569         n->node.key = n;
 570         n->name = memdup(nm, nmlen);
 571         if(!n->name) {
 572                 free(n);
 573                 return NULL;
 574         }
 575         n->namelen = nmlen;
 576         n->namelabs = dname_count_labels(nm);
 577         if(!rbtree_insert(&z->data, &n->node)) {
 578                 log_warn("duplicate auth domain name");
 579                 free(n->name);
 580                 free(n);
 581                 return NULL;
 582         }
 583         return n;
 584 }
 585
 586 /** find domain with exactly the given name */
 587 static struct auth_data*
 588 az_find_name(struct auth_zone* z, uint8_t* nm, size_t nmlen)
 589 {
 590         struct auth_zone key;
 591         key.node.key = &key;
 592         key.name = nm;
 593         key.namelen = nmlen;
 594         key.namelabs = dname_count_labels(nm);
 595         return (struct auth_data*)rbtree_search(&z->data, &key);
 596 }
 597
 598 /** Find domain name (or closest match) */
 599 static void
 600 az_find_domain(struct auth_zone* z, struct query_info* qinfo, int* node_exact,
 601         struct auth_data** node)
 602 {
 603         struct auth_zone key;
 604         key.node.key = &key;
 605         key.name = qinfo->qname;
 606         key.namelen = qinfo->qname_len;
 607         key.namelabs = dname_count_labels(key.name);
 608         *node_exact = rbtree_find_less_equal(&z->data, &key,
 609                 (rbnode_type**)node);
 610 }
 611
 612 /** find or create domain with name in zone */
 613 static struct auth_data*
 614 az_domain_find_or_create(struct auth_zone* z, uint8_t* dname,
 615         size_t dname_len)
 616 {
 617         struct auth_data* n = az_find_name(z, dname, dname_len);
 618         if(!n) {
 619                 n = az_domain_create(z, dname, dname_len);
 620         }
 621         return n;
 622 }
 623
 624 /** find rrset of given type in the domain */
 625 static struct auth_rrset*
 626 az_domain_rrset(struct auth_data* n, uint16_t t)
 627 {
 628         struct auth_rrset* rrset;
 629         if(!n) return NULL;
 630         rrset = n->rrsets;
 631         while(rrset) {
 632                 if(rrset->type == t)
 633                         return rrset;
 634                 rrset = rrset->next;
 635         }
 636         return NULL;
 637 }
 638
 639 /** remove rrset of this type from domain */
 640 static void
 641 domain_remove_rrset(struct auth_data* node, uint16_t rr_type)
 642 {
 643         struct auth_rrset* rrset, *prev;
 644         if(!node) return;
 645         prev = NULL;
 646         rrset = node->rrsets;
 647         while(rrset) {
 648                 if(rrset->type == rr_type) {
 649                         /* found it, now delete it */
 650                         if(prev) prev->next = rrset->next;
 651                         else    node->rrsets = rrset->next;
 652                         auth_rrset_delete(rrset);
 653                         return;
 654                 }
 655                 prev = rrset;
 656                 rrset = rrset->next;
 657         }
 658 }
 659
 660 /** find an rr index in the rrset.  returns true if found */
 661 static int
 662 az_rrset_find_rr(struct packed_rrset_data* d, uint8_t* rdata, size_t len,
 663         size_t* index)
 664 {
 665         size_t i;
 666         for(i=0; i<d->count; i++) {
 667                 if(d->rr_len[i] != len)
 668                         continue;
 669                 if(memcmp(d->rr_data[i], rdata, len) == 0) {
 670                         *index = i;
 671                         return 1;
 672                 }
 673         }
 674         return 0;
 675 }
 676
 677 /** find an rrsig index in the rrset.  returns true if found */
 678 static int
 679 az_rrset_find_rrsig(struct packed_rrset_data* d, uint8_t* rdata, size_t len,
 680         size_t* index)
 681 {
 682         size_t i;
 683         for(i=d->count; i<d->count + d->rrsig_count; i++) {
 684                 if(d->rr_len[i] != len)
 685                         continue;
 686                 if(memcmp(d->rr_data[i], rdata, len) == 0) {
 687                         *index = i;
 688                         return 1;
 689                 }
 690         }
 691         return 0;
 692 }
 693
 694 /** see if rdata is duplicate */
 695 static int
 696 rdata_duplicate(struct packed_rrset_data* d, uint8_t* rdata, size_t len)
 697 {
 698         size_t i;
 699         for(i=0; i<d->count + d->rrsig_count; i++) {
 700                 if(d->rr_len[i] != len)
 701                         continue;
 702                 if(memcmp(d->rr_data[i], rdata, len) == 0)
 703                         return 1;
 704         }
 705         return 0;
 706 }
 707
 708 /** get rrsig type covered from rdata.
 709  * @param rdata: rdata in wireformat, starting with 16bit rdlength.
 710  * @param rdatalen: length of rdata buffer.
 711  * @return type covered (or 0).
 712  */
 713 static uint16_t
 714 rrsig_rdata_get_type_covered(uint8_t* rdata, size_t rdatalen)
 715 {
 716         if(rdatalen < 4)
 717                 return 0;
 718         return sldns_read_uint16(rdata+2);
 719 }
 720
 721 /** remove RR from existing RRset. Also sig, if it is a signature.
 722  * reallocates the packed rrset for a new one, false on alloc failure */
 723 static int
 724 rrset_remove_rr(struct auth_rrset* rrset, size_t index)
 725 {
 726         struct packed_rrset_data* d, *old = rrset->data;
 727         size_t i;
 728         if(index >= old->count + old->rrsig_count)
 729                 return 0; /* index out of bounds */
 730         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old) - (
 731                 sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t) +
 732                 old->rr_len[index]));
 733         if(!d) {
 734                 log_err("malloc failure");
 735                 return 0;
 736         }
 737         d->ttl = old->ttl;
 738         d->count = old->count;
 739         d->rrsig_count = old->rrsig_count;
 740         if(index < d->count) d->count--;
 741         else d->rrsig_count--;
 742         d->trust = old->trust;
 743         d->security = old->security;
 744
 745         /* set rr_len, needed for ptr_fixup */
 746         d->rr_len = (size_t*)((uint8_t*)d +
 747                 sizeof(struct packed_rrset_data));
 748         if(index > 0)
 749                 memmove(d->rr_len, old->rr_len, (index)*sizeof(size_t));
 750         if(index+1 < old->count+old->rrsig_count)
 751                 memmove(&d->rr_len[index], &old->rr_len[index+1],
 752                 (old->count+old->rrsig_count - (index+1))*sizeof(size_t));
 753         packed_rrset_ptr_fixup(d);
 754
 755         /* move over ttls */
 756         if(index > 0)
 757                 memmove(d->rr_ttl, old->rr_ttl, (index)*sizeof(time_t));
 758         if(index+1 < old->count+old->rrsig_count)
 759                 memmove(&d->rr_ttl[index], &old->rr_ttl[index+1],
 760                 (old->count+old->rrsig_count - (index+1))*sizeof(time_t));
 761
 762         /* move over rr_data */
 763         for(i=0; i<d->count+d->rrsig_count; i++) {
 764                 size_t oldi;
 765                 if(i < index) oldi = i;
 766                 else oldi = i+1;
 767                 memmove(d->rr_data[i], old->rr_data[oldi], d->rr_len[i]);
 768         }
 769
 770         /* recalc ttl (lowest of remaining RR ttls) */
 771         if(d->count + d->rrsig_count > 0)
 772                 d->ttl = d->rr_ttl[0];
 773         for(i=0; i<d->count+d->rrsig_count; i++) {
 774                 if(d->rr_ttl[i] < d->ttl)
 775                         d->ttl = d->rr_ttl[i];
 776         }
 777
 778         free(rrset->data);
 779         rrset->data = d;
 780         return 1;
 781 }
 782
 783 /** add RR to existing RRset. If insert_sig is true, add to rrsigs.
 784  * This reallocates the packed rrset for a new one */
 785 static int
 786 rrset_add_rr(struct auth_rrset* rrset, uint32_t rr_ttl, uint8_t* rdata,
 787         size_t rdatalen, int insert_sig)
 788 {
 789         struct packed_rrset_data* d, *old = rrset->data;
 790         size_t total, old_total;
 791
 792         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
 793                 + sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t)
 794                 + rdatalen);
 795         if(!d) {
 796                 log_err("out of memory");
 797                 return 0;
 798         }
 799         /* copy base values */
 800         memcpy(d, old, sizeof(struct packed_rrset_data));
 801         if(!insert_sig) {
 802                 d->count++;
 803         } else {
 804                 d->rrsig_count++;
 805         }
 806         old_total = old->count + old->rrsig_count;
 807         total = d->count + d->rrsig_count;
 808         /* set rr_len, needed for ptr_fixup */
 809         d->rr_len = (size_t*)((uint8_t*)d +
 810                 sizeof(struct packed_rrset_data));
 811         if(old->count != 0)
 812                 memmove(d->rr_len, old->rr_len, old->count*sizeof(size_t));
 813         if(old->rrsig_count != 0)
 814                 memmove(d->rr_len+d->count, old->rr_len+old->count,
 815                         old->rrsig_count*sizeof(size_t));
 816         if(!insert_sig)
 817                 d->rr_len[d->count-1] = rdatalen;
 818         else    d->rr_len[total-1] = rdatalen;
 819         packed_rrset_ptr_fixup(d);
 820         if((time_t)rr_ttl < d->ttl)
 821                 d->ttl = rr_ttl;
 822
 823         /* copy old values into new array */
 824         if(old->count != 0) {
 825                 memmove(d->rr_ttl, old->rr_ttl, old->count*sizeof(time_t));
 826                 /* all the old rr pieces are allocated sequential, so we
 827                  * can copy them in one go */
 828                 memmove(d->rr_data[0], old->rr_data[0],
 829                         (old->rr_data[old->count-1] - old->rr_data[0]) +
 830                         old->rr_len[old->count-1]);
 831         }
 832         if(old->rrsig_count != 0) {
 833                 memmove(d->rr_ttl+d->count, old->rr_ttl+old->count,
 834                         old->rrsig_count*sizeof(time_t));
 835                 memmove(d->rr_data[d->count], old->rr_data[old->count],
 836                         (old->rr_data[old_total-1] - old->rr_data[old->count]) +
 837                         old->rr_len[old_total-1]);
 838         }
 839
 840         /* insert new value */
 841         if(!insert_sig) {
 842                 d->rr_ttl[d->count-1] = rr_ttl;
 843                 memmove(d->rr_data[d->count-1], rdata, rdatalen);
 844         } else {
 845                 d->rr_ttl[total-1] = rr_ttl;
 846                 memmove(d->rr_data[total-1], rdata, rdatalen);
 847         }
 848
 849         rrset->data = d;
 850         free(old);
 851         return 1;
 852 }
 853
 854 /** Create new rrset for node with packed rrset with one RR element */
 855 static struct auth_rrset*
 856 rrset_create(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
 857         uint8_t* rdata, size_t rdatalen)
 858 {
 859         struct auth_rrset* rrset = (struct auth_rrset*)calloc(1,
 860                 sizeof(*rrset));
 861         struct auth_rrset* p, *prev;
 862         struct packed_rrset_data* d;
 863         if(!rrset) {
 864                 log_err("out of memory");
 865                 return NULL;
 866         }
 867         rrset->type = rr_type;
 868
 869         /* the rrset data structure, with one RR */
 870         d = (struct packed_rrset_data*)calloc(1,
 871                 sizeof(struct packed_rrset_data) + sizeof(size_t) +
 872                 sizeof(uint8_t*) + sizeof(time_t) + rdatalen);
 873         if(!d) {
 874                 free(rrset);
 875                 log_err("out of memory");
 876                 return NULL;
 877         }
 878         rrset->data = d;
 879         d->ttl = rr_ttl;
 880         d->trust = rrset_trust_prim_noglue;
 881         d->rr_len = (size_t*)((uint8_t*)d + sizeof(struct packed_rrset_data));
 882         d->rr_data = (uint8_t**)&(d->rr_len[1]);
 883         d->rr_ttl = (time_t*)&(d->rr_data[1]);
 884         d->rr_data[0] = (uint8_t*)&(d->rr_ttl[1]);
 885
 886         /* insert the RR */
 887         d->rr_len[0] = rdatalen;
 888         d->rr_ttl[0] = rr_ttl;
 889         memmove(d->rr_data[0], rdata, rdatalen);
 890         d->count++;
 891
 892         /* insert rrset into linked list for domain */
 893         /* find sorted place to link the rrset into the list */
 894         prev = NULL;
 895         p = node->rrsets;
 896         while(p && p->type<=rr_type) {
 897                 prev = p;
 898                 p = p->next;
 899         }
 900         /* so, prev is smaller, and p is larger than rr_type */
 901         rrset->next = p;
 902         if(prev) prev->next = rrset;
 903         else node->rrsets = rrset;
 904         return rrset;
 905 }
 906
 907 /** count number (and size) of rrsigs that cover a type */
 908 static size_t
 909 rrsig_num_that_cover(struct auth_rrset* rrsig, uint16_t rr_type, size_t* sigsz)
 910 {
 911         struct packed_rrset_data* d = rrsig->data;
 912         size_t i, num = 0;
 913         *sigsz = 0;
 914         log_assert(d && rrsig->type == LDNS_RR_TYPE_RRSIG);
 915         for(i=0; i<d->count+d->rrsig_count; i++) {
 916                 if(rrsig_rdata_get_type_covered(d->rr_data[i],
 917                         d->rr_len[i]) == rr_type) {
 918                         num++;
 919                         (*sigsz) += d->rr_len[i];
 920                 }
 921         }
 922         return num;
 923 }
 924
 925 /** See if rrsig set has covered sigs for rrset and move them over */
 926 static int
 927 rrset_moveover_rrsigs(struct auth_data* node, uint16_t rr_type,
 928         struct auth_rrset* rrset, struct auth_rrset* rrsig)
 929 {
 930         size_t sigs, sigsz, i, j, total;
 931         struct packed_rrset_data* sigold = rrsig->data;
 932         struct packed_rrset_data* old = rrset->data;
 933         struct packed_rrset_data* d, *sigd;
 934
 935         log_assert(rrset->type == rr_type);
 936         log_assert(rrsig->type == LDNS_RR_TYPE_RRSIG);
 937         sigs = rrsig_num_that_cover(rrsig, rr_type, &sigsz);
 938         if(sigs == 0) {
 939                 /* 0 rrsigs to move over, done */
 940                 return 1;
 941         }
 942
 943         /* allocate rrset sigsz larger for extra sigs elements, and
 944          * allocate rrsig sigsz smaller for less sigs elements. */
 945         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
 946                 + sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
 947                 + sigsz);
 948         if(!d) {
 949                 log_err("out of memory");
 950                 return 0;
 951         }
 952         /* copy base values */
 953         total = old->count + old->rrsig_count;
 954         memcpy(d, old, sizeof(struct packed_rrset_data));
 955         d->rrsig_count += sigs;
 956         /* setup rr_len */
 957         d->rr_len = (size_t*)((uint8_t*)d +
 958                 sizeof(struct packed_rrset_data));
 959         if(total != 0)
 960                 memmove(d->rr_len, old->rr_len, total*sizeof(size_t));
 961         j = d->count+d->rrsig_count-sigs;
 962         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
 963                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
 964                         sigold->rr_len[i]) == rr_type) {
 965                         d->rr_len[j] = sigold->rr_len[i];
 966                         j++;
 967                 }
 968         }
 969         packed_rrset_ptr_fixup(d);
 970
 971         /* copy old values into new array */
 972         if(total != 0) {
 973                 memmove(d->rr_ttl, old->rr_ttl, total*sizeof(time_t));
 974                 /* all the old rr pieces are allocated sequential, so we
 975                  * can copy them in one go */
 976                 memmove(d->rr_data[0], old->rr_data[0],
 977                         (old->rr_data[total-1] - old->rr_data[0]) +
 978                         old->rr_len[total-1]);
 979         }
 980
 981         /* move over the rrsigs to the larger rrset*/
 982         j = d->count+d->rrsig_count-sigs;
 983         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
 984                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
 985                         sigold->rr_len[i]) == rr_type) {
 986                         /* move this one over to location j */
 987                         d->rr_ttl[j] = sigold->rr_ttl[i];
 988                         memmove(d->rr_data[j], sigold->rr_data[i],
 989                                 sigold->rr_len[i]);
 990                         if(d->rr_ttl[j] < d->ttl)
 991                                 d->ttl = d->rr_ttl[j];
 992                         j++;
 993                 }
 994         }
 995
 996         /* put it in and deallocate the old rrset */
 997         rrset->data = d;
 998         free(old);
 999
1000         /* now make rrsig set smaller */
1001         if(sigold->count+sigold->rrsig_count == sigs) {
1002                 /* remove all sigs from rrsig, remove it entirely */
1003                 domain_remove_rrset(node, LDNS_RR_TYPE_RRSIG);
1004                 return 1;
1005         }
1006         log_assert(packed_rrset_sizeof(sigold) > sigs*(sizeof(size_t) +
1007                 sizeof(uint8_t*) + sizeof(time_t)) + sigsz);
1008         sigd = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(sigold)
1009                 - sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
1010                 - sigsz);
1011         if(!sigd) {
1012                 /* no need to free up d, it has already been placed in the
1013                  * node->rrset structure */
1014                 log_err("out of memory");
1015                 return 0;
1016         }
1017         /* copy base values */
1018         memcpy(sigd, sigold, sizeof(struct packed_rrset_data));
1019         /* in sigd the RRSIGs are stored in the base of the RR, in count */
1020         sigd->count -= sigs;
1021         /* setup rr_len */
1022         sigd->rr_len = (size_t*)((uint8_t*)sigd +
1023                 sizeof(struct packed_rrset_data));
1024         j = 0;
1025         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1026                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1027                         sigold->rr_len[i]) != rr_type) {
1028                         sigd->rr_len[j] = sigold->rr_len[i];
1029                         j++;
1030                 }
1031         }
1032         packed_rrset_ptr_fixup(sigd);
1033
1034         /* copy old values into new rrsig array */
1035         j = 0;
1036         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1037                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1038                         sigold->rr_len[i]) != rr_type) {
1039                         /* move this one over to location j */
1040                         sigd->rr_ttl[j] = sigold->rr_ttl[i];
1041                         memmove(sigd->rr_data[j], sigold->rr_data[i],
1042                                 sigold->rr_len[i]);
1043                         if(j==0) sigd->ttl = sigd->rr_ttl[j];
1044                         else {
1045                                 if(sigd->rr_ttl[j] < sigd->ttl)
1046                                         sigd->ttl = sigd->rr_ttl[j];
1047                         }
1048                         j++;
1049                 }
1050         }
1051
1052         /* put it in and deallocate the old rrset */
1053         rrsig->data = sigd;
1054         free(sigold);
1055
1056         return 1;
1057 }
1058
1059 /** copy the rrsigs from the rrset to the rrsig rrset, because the rrset
1060  * is going to be deleted.  reallocates the RRSIG rrset data. */
1061 static int
1062 rrsigs_copy_from_rrset_to_rrsigset(struct auth_rrset* rrset,
1063         struct auth_rrset* rrsigset)
1064 {
1065         size_t i;
1066         if(rrset->data->rrsig_count == 0)
1067                 return 1;
1068
1069         /* move them over one by one, because there might be duplicates,
1070          * duplicates are ignored */
1071         for(i=rrset->data->count;
1072                 i<rrset->data->count+rrset->data->rrsig_count; i++) {
1073                 uint8_t* rdata = rrset->data->rr_data[i];
1074                 size_t rdatalen = rrset->data->rr_len[i];
1075                 time_t rr_ttl  = rrset->data->rr_ttl[i];
1076
1077                 if(rdata_duplicate(rrsigset->data, rdata, rdatalen)) {
1078                         continue;
1079                 }
1080                 if(!rrset_add_rr(rrsigset, rr_ttl, rdata, rdatalen, 0))
1081                         return 0;
1082         }
1083         return 1;
1084 }
1085
1086 /** Add rr to node, ignores duplicate RRs,
1087  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1088 static int
1089 az_domain_add_rr(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
1090         uint8_t* rdata, size_t rdatalen, int* duplicate)
1091 {
1092         struct auth_rrset* rrset;
1093         /* packed rrsets have their rrsigs along with them, sort them out */
1094         if(rr_type == LDNS_RR_TYPE_RRSIG) {
1095                 uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1096                 if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1097                         /* a node of the correct type exists, add the RRSIG
1098                          * to the rrset of the covered data type */
1099                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1100                                 if(duplicate) *duplicate = 1;
1101                                 return 1;
1102                         }
1103                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 1))
1104                                 return 0;
1105                 } else if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1106                         /* add RRSIG to rrset of type RRSIG */
1107                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1108                                 if(duplicate) *duplicate = 1;
1109                                 return 1;
1110                         }
1111                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1112                                 return 0;
1113                 } else {
1114                         /* create rrset of type RRSIG */
1115                         if(!rrset_create(node, rr_type, rr_ttl, rdata,
1116                                 rdatalen))
1117                                 return 0;
1118                 }
1119         } else {
1120                 /* normal RR type */
1121                 if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1122                         /* add data to existing node with data type */
1123                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1124                                 if(duplicate) *duplicate = 1;
1125                                 return 1;
1126                         }
1127                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1128                                 return 0;
1129                 } else {
1130                         struct auth_rrset* rrsig;
1131                         /* create new node with data type */
1132                         if(!(rrset=rrset_create(node, rr_type, rr_ttl, rdata,
1133                                 rdatalen)))
1134                                 return 0;
1135
1136                         /* see if node of type RRSIG has signatures that
1137                          * cover the data type, and move them over */
1138                         /* and then make the RRSIG type smaller */
1139                         if((rrsig=az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
1140                                 != NULL) {
1141                                 if(!rrset_moveover_rrsigs(node, rr_type,
1142                                         rrset, rrsig))
1143                                         return 0;
1144                         }
1145                 }
1146         }
1147         return 1;
1148 }
1149
1150 /** insert RR into zone, ignore duplicates */
1151 static int
1152 az_insert_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1153         size_t dname_len, int* duplicate)
1154 {
1155         struct auth_data* node;
1156         uint8_t* dname = rr;
1157         uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1158         uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1159         uint32_t rr_ttl = sldns_wirerr_get_ttl(rr, rr_len, dname_len);
1160         size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1161                 dname_len))+2;
1162         /* rdata points to rdata prefixed with uint16 rdatalength */
1163         uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1164
1165         if(rr_class != z->dclass) {
1166                 log_err("wrong class for RR");
1167                 return 0;
1168         }
1169         if(!(node=az_domain_find_or_create(z, dname, dname_len))) {
1170                 log_err("cannot create domain");
1171                 return 0;
1172         }
1173         if(!az_domain_add_rr(node, rr_type, rr_ttl, rdata, rdatalen,
1174                 duplicate)) {
1175                 log_err("cannot add RR to domain");
1176                 return 0;
1177         }
1178         return 1;
1179 }
1180
1181 /** Remove rr from node, ignores nonexisting RRs,
1182  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1183 static int
1184 az_domain_remove_rr(struct auth_data* node, uint16_t rr_type,
1185         uint8_t* rdata, size_t rdatalen, int* nonexist)
1186 {
1187         struct auth_rrset* rrset;
1188         size_t index = 0;
1189
1190         /* find the plain RR of the given type */
1191         if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1192                 if(az_rrset_find_rr(rrset->data, rdata, rdatalen, &index)) {
1193                         if(rrset->data->count == 1 &&
1194                                 rrset->data->rrsig_count == 0) {
1195                                 /* last RR, delete the rrset */
1196                                 domain_remove_rrset(node, rr_type);
1197                         } else if(rrset->data->count == 1 &&
1198                                 rrset->data->rrsig_count != 0) {
1199                                 /* move RRSIGs to the RRSIG rrset, or
1200                                  * this one becomes that RRset */
1201                                 struct auth_rrset* rrsigset = az_domain_rrset(
1202                                         node, LDNS_RR_TYPE_RRSIG);
1203                                 if(rrsigset) {
1204                                         /* move left over rrsigs to the
1205                                          * existing rrset of type RRSIG */
1206                                         rrsigs_copy_from_rrset_to_rrsigset(
1207                                                 rrset, rrsigset);
1208                                         /* and then delete the rrset */
1209                                         domain_remove_rrset(node, rr_type);
1210                                 } else {
1211                                         /* no rrset of type RRSIG, this
1212                                          * set is now of that type,
1213                                          * just remove the rr */
1214                                         if(!rrset_remove_rr(rrset, index))
1215                                                 return 0;
1216                                         rrset->type = LDNS_RR_TYPE_RRSIG;
1217                                         rrset->data->count = rrset->data->rrsig_count;
1218                                         rrset->data->rrsig_count = 0;
1219                                 }
1220                         } else {
1221                                 /* remove the RR from the rrset */
1222                                 if(!rrset_remove_rr(rrset, index))
1223                                         return 0;
1224                         }
1225                         return 1;
1226                 }
1227                 /* rr not found in rrset */
1228         }
1229
1230         /* is it a type RRSIG, look under the covered type */
1231         if(rr_type == LDNS_RR_TYPE_RRSIG) {
1232                 uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1233                 if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1234                         if(az_rrset_find_rrsig(rrset->data, rdata, rdatalen,
1235                                 &index)) {
1236                                 /* rrsig should have d->count > 0, be
1237                                  * over some rr of that type */
1238                                 /* remove the rrsig from the rrsigs list of the
1239                                  * rrset */
1240                                 if(!rrset_remove_rr(rrset, index))
1241                                         return 0;
1242                                 return 1;
1243                         }
1244                 }
1245                 /* also RRSIG not found */
1246         }
1247
1248         /* nothing found to delete */
1249         if(nonexist) *nonexist = 1;
1250         return 1;
1251 }
1252
1253 /** remove RR from zone, ignore if it does not exist, false on alloc failure*/
1254 static int
1255 az_remove_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1256         size_t dname_len, int* nonexist)
1257 {
1258         struct auth_data* node;
1259         uint8_t* dname = rr;
1260         uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1261         uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1262         size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1263                 dname_len))+2;
1264         /* rdata points to rdata prefixed with uint16 rdatalength */
1265         uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1266
1267         if(rr_class != z->dclass) {
1268                 log_err("wrong class for RR");
1269                 /* really also a nonexisting entry, because no records
1270                  * of that class in the zone, but return an error because
1271                  * getting records of the wrong class is a failure of the
1272                  * zone transfer */
1273                 return 0;
1274         }
1275         node = az_find_name(z, dname, dname_len);
1276         if(!node) {
1277                 /* node with that name does not exist */
1278                 /* nonexisting entry, because no such name */
1279                 *nonexist = 1;
1280                 return 1;
1281         }
1282         if(!az_domain_remove_rr(node, rr_type, rdata, rdatalen, nonexist)) {
1283                 /* alloc failure or so */
1284                 return 0;
1285         }
1286         /* remove the node, if necessary */
1287         /* an rrsets==NULL entry is not kept around for empty nonterminals,
1288          * and also parent nodes are not kept around, so we just delete it */
1289         if(node->rrsets == NULL) {
1290                 (void)rbtree_delete(&z->data, node);
1291                 auth_data_delete(node);
1292         }
1293         return 1;
1294 }
1295
1296 /** decompress an RR into the buffer where it'll be an uncompressed RR
1297  * with uncompressed dname and uncompressed rdata (dnames) */
1298 static int
1299 decompress_rr_into_buffer(struct sldns_buffer* buf, uint8_t* pkt,
1300         size_t pktlen, uint8_t* dname, uint16_t rr_type, uint16_t rr_class,
1301         uint32_t rr_ttl, uint8_t* rr_data, uint16_t rr_rdlen)
1302 {
1303         sldns_buffer pktbuf;
1304         size_t dname_len = 0;
1305         size_t rdlenpos;
1306         size_t rdlen;
1307         uint8_t* rd;
1308         const sldns_rr_descriptor* desc;
1309         sldns_buffer_init_frm_data(&pktbuf, pkt, pktlen);
1310         sldns_buffer_clear(buf);
1311
1312         /* decompress dname */
1313         sldns_buffer_set_position(&pktbuf,
1314                 (size_t)(dname - sldns_buffer_current(&pktbuf)));
1315         dname_len = pkt_dname_len(&pktbuf);
1316         if(dname_len == 0) return 0; /* parse fail on dname */
1317         if(!sldns_buffer_available(buf, dname_len)) return 0;
1318         dname_pkt_copy(&pktbuf, sldns_buffer_current(buf), dname);
1319         sldns_buffer_skip(buf, (ssize_t)dname_len);
1320
1321         /* type, class, ttl and rdatalength fields */
1322         if(!sldns_buffer_available(buf, 10)) return 0;
1323         sldns_buffer_write_u16(buf, rr_type);
1324         sldns_buffer_write_u16(buf, rr_class);
1325         sldns_buffer_write_u32(buf, rr_ttl);
1326         rdlenpos = sldns_buffer_position(buf);
1327         sldns_buffer_write_u16(buf, 0); /* rd length position */
1328
1329         /* decompress rdata */
1330         desc = sldns_rr_descript(rr_type);
1331         rd = rr_data;
1332         rdlen = rr_rdlen;
1333         if(rdlen > 0 && desc && desc->_dname_count > 0) {
1334                 int count = (int)desc->_dname_count;
1335                 int rdf = 0;
1336                 size_t len; /* how much rdata to plain copy */
1337                 size_t uncompressed_len, compressed_len;
1338                 size_t oldpos;
1339                 /* decompress dnames. */
1340                 while(rdlen > 0 && count) {
1341                         switch(desc->_wireformat[rdf]) {
1342                         case LDNS_RDF_TYPE_DNAME:
1343                                 sldns_buffer_set_position(&pktbuf,
1344                                         (size_t)(rd -
1345                                         sldns_buffer_begin(&pktbuf)));
1346                                 oldpos = sldns_buffer_position(&pktbuf);
1347                                 /* moves pktbuf to right after the
1348                                  * compressed dname, and returns uncompressed
1349                                  * dname length */
1350                                 uncompressed_len = pkt_dname_len(&pktbuf);
1351                                 if(!uncompressed_len)
1352                                         return 0; /* parse error in dname */
1353                                 if(!sldns_buffer_available(buf,
1354                                         uncompressed_len))
1355                                         /* dname too long for buffer */
1356                                         return 0;
1357                                 dname_pkt_copy(&pktbuf,
1358                                         sldns_buffer_current(buf), rd);
1359                                 sldns_buffer_skip(buf, (ssize_t)uncompressed_len);
1360                                 compressed_len = sldns_buffer_position(
1361                                         &pktbuf) - oldpos;
1362                                 rd += compressed_len;
1363                                 rdlen -= compressed_len;
1364                                 count--;
1365                                 len = 0;
1366                                 break;
1367                         case LDNS_RDF_TYPE_STR:
1368                                 len = rd[0] + 1;
1369                                 break;
1370                         default:
1371                                 len = get_rdf_size(desc->_wireformat[rdf]);
1372                                 break;
1373                         }
1374                         if(len) {
1375                                 if(!sldns_buffer_available(buf, len))
1376                                         return 0; /* too long for buffer */
1377                                 sldns_buffer_write(buf, rd, len);
1378                                 rd += len;
1379                                 rdlen -= len;
1380                         }
1381                         rdf++;
1382                 }
1383         }
1384         /* copy remaining data */
1385         if(rdlen > 0) {
1386                 if(!sldns_buffer_available(buf, rdlen)) return 0;
1387                 sldns_buffer_write(buf, rd, rdlen);
1388         }
1389         /* fixup rdlength */
1390         sldns_buffer_write_u16_at(buf, rdlenpos,
1391                 sldns_buffer_position(buf)-rdlenpos-2);
1392         sldns_buffer_flip(buf);
1393         return 1;
1394 }
1395
1396 /** insert RR into zone, from packet, decompress RR,
1397  * if duplicate is nonNULL set the flag but otherwise ignore duplicates */
1398 static int
1399 az_insert_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1400         struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1401         uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1402         uint16_t rr_rdlen, int* duplicate)
1403 {
1404         uint8_t* rr;
1405         size_t rr_len;
1406         size_t dname_len;
1407         if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1408                 rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1409                 log_err("could not decompress RR");
1410                 return 0;
1411         }
1412         rr = sldns_buffer_begin(scratch_buffer);
1413         rr_len = sldns_buffer_limit(scratch_buffer);
1414         dname_len = dname_valid(rr, rr_len);
1415         return az_insert_rr(z, rr, rr_len, dname_len, duplicate);
1416 }
1417
1418 /** remove RR from zone, from packet, decompress RR,
1419  * if nonexist is nonNULL set the flag but otherwise ignore nonexisting entries*/
1420 static int
1421 az_remove_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1422         struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1423         uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1424         uint16_t rr_rdlen, int* nonexist)
1425 {
1426         uint8_t* rr;
1427         size_t rr_len;
1428         size_t dname_len;
1429         if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1430                 rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1431                 log_err("could not decompress RR");
1432                 return 0;
1433         }
1434         rr = sldns_buffer_begin(scratch_buffer);
1435         rr_len = sldns_buffer_limit(scratch_buffer);
1436         dname_len = dname_valid(rr, rr_len);
1437         return az_remove_rr(z, rr, rr_len, dname_len, nonexist);
1438 }
1439
1440 /**
1441  * Parse zonefile
1442  * @param z: zone to read in.
1443  * @param in: file to read from (just opened).
1444  * @param rr: buffer to use for RRs, 64k.
1445  *      passed so that recursive includes can use the same buffer and do
1446  *      not grow the stack too much.
1447  * @param rrbuflen: sizeof rr buffer.
1448  * @param state: parse state with $ORIGIN, $TTL and 'prev-dname' and so on,
1449  *      that is kept between includes.
1450  *      The lineno is set at 1 and then increased by the function.
1451  * @param fname: file name.
1452  * @param depth: recursion depth for includes
1453  * returns false on failure, has printed an error message
1454  */
1455 static int
1456 az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
1457         struct sldns_file_parse_state* state, char* fname, int depth)
1458 {
1459         size_t rr_len, dname_len;
1460         int status;
1461         state->lineno = 1;
1462
1463         while(!feof(in)) {
1464                 rr_len = rrbuflen;
1465                 dname_len = 0;
1466                 status = sldns_fp2wire_rr_buf(in, rr, &rr_len, &dname_len,
1467                         state);
1468                 if(status == LDNS_WIREPARSE_ERR_INCLUDE && rr_len == 0) {
1469                         /* we have $INCLUDE or $something */
1470                         if(strncmp((char*)rr, "$INCLUDE ", 9) == 0 ||
1471                            strncmp((char*)rr, "$INCLUDE\t", 9) == 0) {
1472                                 FILE* inc;
1473                                 int lineno_orig = state->lineno;
1474                                 char* incfile = (char*)rr + 8;
1475                                 if(depth > MAX_INCLUDE_DEPTH) {
1476                                         log_err("%s:%d max include depth"
1477                                           "exceeded", fname, state->lineno);
1478                                         return 0;
1479                                 }
1480                                 /* skip spaces */
1481                                 while(*incfile == ' ' || *incfile == '\t')
1482                                         incfile++;
1483                                 incfile = strdup(incfile);
1484                                 if(!incfile) {
1485                                         log_err("malloc failure");
1486                                         return 0;
1487                                 }
1488                                 verbose(VERB_ALGO, "opening $INCLUDE %s",
1489                                         incfile);
1490                                 inc = fopen(incfile, "r");
1491                                 if(!inc) {
1492                                         log_err("%s:%d cannot open include "
1493                                                 "file %s: %s", z->zonefile,
1494                                                 lineno_orig, incfile,
1495                                                 strerror(errno));
1496                                         free(incfile);
1497                                         return 0;
1498                                 }
1499                                 /* recurse read that file now */
1500                                 if(!az_parse_file(z, inc, rr, rrbuflen,
1501                                         state, incfile, depth+1)) {
1502                                         log_err("%s:%d cannot parse include "
1503                                                 "file %s", fname,
1504                                                 lineno_orig, incfile);
1505                                         fclose(inc);
1506                                         free(incfile);
1507                                         return 0;
1508                                 }
1509                                 fclose(inc);
1510                                 verbose(VERB_ALGO, "done with $INCLUDE %s",
1511                                         incfile);
1512                                 free(incfile);
1513                                 state->lineno = lineno_orig;
1514                         }
1515                         continue;
1516                 }
1517                 if(status != 0) {
1518                         log_err("parse error %s %d:%d: %s", fname,
1519                                 state->lineno, LDNS_WIREPARSE_OFFSET(status),
1520                                 sldns_get_errorstr_parse(status));
1521                         return 0;
1522                 }
1523                 if(rr_len == 0) {
1524                         /* EMPTY line, TTL or ORIGIN */
1525                         continue;
1526                 }
1527                 /* insert wirerr in rrbuf */
1528                 if(!az_insert_rr(z, rr, rr_len, dname_len, NULL)) {
1529                         char buf[17];
1530                         sldns_wire2str_type_buf(sldns_wirerr_get_type(rr,
1531                                 rr_len, dname_len), buf, sizeof(buf));
1532                         log_err("%s:%d cannot insert RR of type %s",
1533                                 fname, state->lineno, buf);
1534                         return 0;
1535                 }
1536         }
1537         return 1;
1538 }
1539
1540 int
1541 auth_zone_read_zonefile(struct auth_zone* z)
1542 {
1543         uint8_t rr[LDNS_RR_BUF_SIZE];
1544         struct sldns_file_parse_state state;
1545         FILE* in;
1546         if(!z || !z->zonefile || z->zonefile[0]==0)
1547                 return 1; /* no file, or "", nothing to read */
1548         if(verbosity >= VERB_ALGO) {
1549                 char nm[255+1];
1550                 dname_str(z->name, nm);
1551                 verbose(VERB_ALGO, "read zonefile %s for %s", z->zonefile, nm);
1552         }
1553         in = fopen(z->zonefile, "r");
1554         if(!in) {
1555                 char* n = sldns_wire2str_dname(z->name, z->namelen);
1556                 if(z->zone_is_slave && errno == ENOENT) {
1557                         /* we fetch the zone contents later, no file yet */
1558                         verbose(VERB_ALGO, "no zonefile %s for %s",
1559                                 z->zonefile, n?n:"error");
1560                         free(n);
1561                         return 1;
1562                 }
1563                 log_err("cannot open zonefile %s for %s: %s",
1564                         z->zonefile, n?n:"error", strerror(errno));
1565                 free(n);
1566                 return 0;
1567         }
1568
1569         /* clear the data tree */
1570         traverse_postorder(&z->data, auth_data_del, NULL);
1571         rbtree_init(&z->data, &auth_data_cmp);
1572
1573         memset(&state, 0, sizeof(state));
1574         /* default TTL to 3600 */
1575         state.default_ttl = 3600;
1576         /* set $ORIGIN to the zone name */
1577         if(z->namelen <= sizeof(state.origin)) {
1578                 memcpy(state.origin, z->name, z->namelen);
1579                 state.origin_len = z->namelen;
1580         }
1581         /* parse the (toplevel) file */
1582         if(!az_parse_file(z, in, rr, sizeof(rr), &state, z->zonefile, 0)) {
1583                 char* n = sldns_wire2str_dname(z->name, z->namelen);
1584                 log_err("error parsing zonefile %s for %s",
1585                         z->zonefile, n?n:"error");
1586                 free(n);
1587                 fclose(in);
1588                 return 0;
1589         }
1590         fclose(in);
1591         return 1;
1592 }
1593
1594 /** write buffer to file and check return codes */
1595 static int
1596 write_out(FILE* out, const char* str, size_t len)
1597 {
1598         size_t r;
1599         if(len == 0)
1600                 return 1;
1601         r = fwrite(str, 1, len, out);
1602         if(r == 0) {
1603                 log_err("write failed: %s", strerror(errno));
1604                 return 0;
1605         } else if(r < len) {
1606                 log_err("write failed: too short (disk full?)");
1607                 return 0;
1608         }
1609         return 1;
1610 }
1611
1612 /** convert auth rr to string */
1613 static int
1614 auth_rr_to_string(uint8_t* nm, size_t nmlen, uint16_t tp, uint16_t cl,
1615         struct packed_rrset_data* data, size_t i, char* s, size_t buflen)
1616 {
1617         int w = 0;
1618         size_t slen = buflen, datlen;
1619         uint8_t* dat;
1620         if(i >= data->count) tp = LDNS_RR_TYPE_RRSIG;
1621         dat = nm;
1622         datlen = nmlen;
1623         w += sldns_wire2str_dname_scan(&dat, &datlen, &s, &slen, NULL, 0);
1624         w += sldns_str_print(&s, &slen, "\t");
1625         w += sldns_str_print(&s, &slen, "%lu\t", (unsigned long)data->rr_ttl[i]);
1626         w += sldns_wire2str_class_print(&s, &slen, cl);
1627         w += sldns_str_print(&s, &slen, "\t");
1628         w += sldns_wire2str_type_print(&s, &slen, tp);
1629         w += sldns_str_print(&s, &slen, "\t");
1630         datlen = data->rr_len[i]-2;
1631         dat = data->rr_data[i]+2;
1632         w += sldns_wire2str_rdata_scan(&dat, &datlen, &s, &slen, tp, NULL, 0);
1633
1634         if(tp == LDNS_RR_TYPE_DNSKEY) {
1635                 w += sldns_str_print(&s, &slen, " ;{id = %u}",
1636                         sldns_calc_keytag_raw(data->rr_data[i]+2,
1637                                 data->rr_len[i]-2));
1638         }
1639         w += sldns_str_print(&s, &slen, "\n");
1640
1641         if(w > (int)buflen) {
1642                 log_nametypeclass(0, "RR too long to print", nm, tp, cl);
1643                 return 0;
1644         }
1645         return 1;
1646 }
1647
1648 /** write rrset to file */
1649 static int
1650 auth_zone_write_rrset(struct auth_zone* z, struct auth_data* node,
1651         struct auth_rrset* r, FILE* out)
1652 {
1653         size_t i, count = r->data->count + r->data->rrsig_count;
1654         char buf[LDNS_RR_BUF_SIZE];
1655         for(i=0; i<count; i++) {
1656                 if(!auth_rr_to_string(node->name, node->namelen, r->type,
1657                         z->dclass, r->data, i, buf, sizeof(buf))) {
1658                         verbose(VERB_ALGO, "failed to rr2str rr %d", (int)i);
1659                         continue;
1660                 }
1661                 if(!write_out(out, buf, strlen(buf)))
1662                         return 0;
1663         }
1664         return 1;
1665 }
1666
1667 /** write domain to file */
1668 static int
1669 auth_zone_write_domain(struct auth_zone* z, struct auth_data* n, FILE* out)
1670 {
1671         struct auth_rrset* r;
1672         /* if this is zone apex, write SOA first */
1673         if(z->namelen == n->namelen) {
1674                 struct auth_rrset* soa = az_domain_rrset(n, LDNS_RR_TYPE_SOA);
1675                 if(soa) {
1676                         if(!auth_zone_write_rrset(z, n, soa, out))
1677                                 return 0;
1678                 }
1679         }
1680         /* write all the RRsets for this domain */
1681         for(r = n->rrsets; r; r = r->next) {
1682                 if(z->namelen == n->namelen &&
1683                         r->type == LDNS_RR_TYPE_SOA)
1684                         continue; /* skip SOA here */
1685                 if(!auth_zone_write_rrset(z, n, r, out))
1686                         return 0;
1687         }
1688         return 1;
1689 }
1690
1691 int auth_zone_write_file(struct auth_zone* z, const char* fname)
1692 {
1693         FILE* out;
1694         struct auth_data* n;
1695         out = fopen(fname, "w");
1696         if(!out) {
1697                 log_err("could not open %s: %s", fname, strerror(errno));
1698                 return 0;
1699         }
1700         RBTREE_FOR(n, struct auth_data*, &z->data) {
1701                 if(!auth_zone_write_domain(z, n, out)) {
1702                         log_err("could not write domain to %s", fname);
1703                         fclose(out);
1704                         return 0;
1705                 }
1706         }
1707         fclose(out);
1708         return 1;
1709 }
1710
1711 /** read all auth zones from file (if they have) */
1712 static int
1713 auth_zones_read_zones(struct auth_zones* az)
1714 {
1715         struct auth_zone* z;
1716         lock_rw_wrlock(&az->lock);
1717         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1718                 lock_rw_wrlock(&z->lock);
1719                 if(!auth_zone_read_zonefile(z)) {
1720                         lock_rw_unlock(&z->lock);
1721                         lock_rw_unlock(&az->lock);
1722                         return 0;
1723                 }
1724                 lock_rw_unlock(&z->lock);
1725         }
1726         lock_rw_unlock(&az->lock);
1727         return 1;
1728 }
1729
1730 /** find serial number of zone or false if none */
1731 int
1732 auth_zone_get_serial(struct auth_zone* z, uint32_t* serial)
1733 {
1734         struct auth_data* apex;
1735         struct auth_rrset* soa;
1736         struct packed_rrset_data* d;
1737         apex = az_find_name(z, z->name, z->namelen);
1738         if(!apex) return 0;
1739         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1740         if(!soa || soa->data->count==0)
1741                 return 0; /* no RRset or no RRs in rrset */
1742         if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1743         d = soa->data;
1744         *serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1745         return 1;
1746 }
1747
1748 /** Find auth_zone SOA and populate the values in xfr(soa values). */
1749 static int
1750 xfr_find_soa(struct auth_zone* z, struct auth_xfer* xfr)
1751 {
1752         struct auth_data* apex;
1753         struct auth_rrset* soa;
1754         struct packed_rrset_data* d;
1755         apex = az_find_name(z, z->name, z->namelen);
1756         if(!apex) return 0;
1757         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1758         if(!soa || soa->data->count==0)
1759                 return 0; /* no RRset or no RRs in rrset */
1760         if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1761         /* SOA record ends with serial, refresh, retry, expiry, minimum,
1762          * as 4 byte fields */
1763         d = soa->data;
1764         xfr->have_zone = 1;
1765         xfr->serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1766         xfr->refresh = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-16));
1767         xfr->retry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-12));
1768         xfr->expiry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-8));
1769         /* soa minimum at d->rr_len[0]-4 */
1770         return 1;
1771 }
1772
1773 /**
1774  * Setup auth_xfer zone
1775  * This populates the have_zone, soa values, and so on times.
1776  * Doesn't do network traffic yet, can set option flags.
1777  * @param z: locked by caller, and modified for setup
1778  * @param x: locked by caller, and modified.
1779  * @return false on failure.
1780  */
1781 static int
1782 auth_xfer_setup(struct auth_zone* z, struct auth_xfer* x)
1783 {
1784         /* for a zone without zone transfers, x==NULL, so skip them,
1785          * i.e. the zone config is fixed with no masters or urls */
1786         if(!z || !x) return 1;
1787         if(!xfr_find_soa(z, x)) {
1788                 return 1;
1789         }
1790         /* nothing for probe, nextprobe and transfer tasks */
1791         return 1;
1792 }
1793
1794 /**
1795  * Setup all zones
1796  * @param az: auth zones structure
1797  * @return false on failure.
1798  */
1799 static int
1800 auth_zones_setup_zones(struct auth_zones* az)
1801 {
1802         struct auth_zone* z;
1803         struct auth_xfer* x;
1804         lock_rw_wrlock(&az->lock);
1805         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1806                 lock_rw_wrlock(&z->lock);
1807                 x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
1808                 if(x) {
1809                         lock_basic_lock(&x->lock);
1810                 }
1811                 if(!auth_xfer_setup(z, x)) {
1812                         if(x) {
1813                                 lock_basic_unlock(&x->lock);
1814                         }
1815                         lock_rw_unlock(&z->lock);
1816                         lock_rw_unlock(&az->lock);
1817                         return 0;
1818                 }
1819                 if(x) {
1820                         lock_basic_unlock(&x->lock);
1821                 }
1822                 lock_rw_unlock(&z->lock);
1823         }
1824         lock_rw_unlock(&az->lock);
1825         return 1;
1826 }
1827
1828 /** set config items and create zones */
1829 static int
1830 auth_zones_cfg(struct auth_zones* az, struct config_auth* c)
1831 {
1832         struct auth_zone* z;
1833         struct auth_xfer* x = NULL;
1834
1835         /* create zone */
1836         lock_rw_wrlock(&az->lock);
1837         if(!(z=auth_zones_find_or_add_zone(az, c->name))) {
1838                 lock_rw_unlock(&az->lock);
1839                 return 0;
1840         }
1841         if(c->masters || c->urls) {
1842                 if(!(x=auth_zones_find_or_add_xfer(az, z))) {
1843                         lock_rw_unlock(&az->lock);
1844                         lock_rw_unlock(&z->lock);
1845                         return 0;
1846                 }
1847         }
1848         if(c->for_downstream)
1849                 az->have_downstream = 1;
1850         lock_rw_unlock(&az->lock);
1851
1852         /* set options */
1853         z->zone_deleted = 0;
1854         if(!auth_zone_set_zonefile(z, c->zonefile)) {
1855                 if(x) {
1856                         lock_basic_unlock(&x->lock);
1857                 }
1858                 lock_rw_unlock(&z->lock);
1859                 return 0;
1860         }
1861         z->for_downstream = c->for_downstream;
1862         z->for_upstream = c->for_upstream;
1863         z->fallback_enabled = c->fallback_enabled;
1864
1865         /* xfer zone */
1866         if(x) {
1867                 z->zone_is_slave = 1;
1868                 /* set options on xfer zone */
1869                 if(!xfer_set_masters(&x->task_probe->masters, c, 0)) {
1870                         lock_basic_unlock(&x->lock);
1871                         lock_rw_unlock(&z->lock);
1872                         return 0;
1873                 }
1874                 if(!xfer_set_masters(&x->task_transfer->masters, c, 1)) {
1875                         lock_basic_unlock(&x->lock);
1876                         lock_rw_unlock(&z->lock);
1877                         return 0;
1878                 }
1879                 lock_basic_unlock(&x->lock);
1880         }
1881
1882         lock_rw_unlock(&z->lock);
1883         return 1;
1884 }
1885
1886 /** set all auth zones deleted, then in auth_zones_cfg, it marks them
1887  * as nondeleted (if they are still in the config), and then later
1888  * we can find deleted zones */
1889 static void
1890 az_setall_deleted(struct auth_zones* az)
1891 {
1892         struct auth_zone* z;
1893         lock_rw_wrlock(&az->lock);
1894         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1895                 lock_rw_wrlock(&z->lock);
1896                 z->zone_deleted = 1;
1897                 lock_rw_unlock(&z->lock);
1898         }
1899         lock_rw_unlock(&az->lock);
1900 }
1901
1902 /** find zones that are marked deleted and delete them.
1903  * This is called from apply_cfg, and there are no threads and no
1904  * workers, so the xfr can just be deleted. */
1905 static void
1906 az_delete_deleted_zones(struct auth_zones* az)
1907 {
1908         struct auth_zone* z;
1909         struct auth_zone* delete_list = NULL, *next;
1910         struct auth_xfer* xfr;
1911         lock_rw_wrlock(&az->lock);
1912         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1913                 lock_rw_wrlock(&z->lock);
1914                 if(z->zone_deleted) {
1915                         /* we cannot alter the rbtree right now, but
1916                          * we can put it on a linked list and then
1917                          * delete it */
1918                         z->delete_next = delete_list;
1919                         delete_list = z;
1920                 }
1921                 lock_rw_unlock(&z->lock);
1922         }
1923         /* now we are out of the tree loop and we can loop and delete
1924          * the zones */
1925         z = delete_list;
1926         while(z) {
1927                 next = z->delete_next;
1928                 xfr = auth_xfer_find(az, z->name, z->namelen, z->dclass);
1929                 if(xfr) {
1930                         (void)rbtree_delete(&az->xtree, &xfr->node);
1931                         auth_xfer_delete(xfr);
1932                 }
1933                 (void)rbtree_delete(&az->ztree, &z->node);
1934                 auth_zone_delete(z);
1935                 z = next;
1936         }
1937         lock_rw_unlock(&az->lock);
1938 }
1939
1940 int auth_zones_apply_cfg(struct auth_zones* az, struct config_file* cfg,
1941         int setup)
1942 {
1943         struct config_auth* p;
1944         az_setall_deleted(az);
1945         for(p = cfg->auths; p; p = p->next) {
1946                 if(!p->name || p->name[0] == 0) {
1947                         log_warn("auth-zone without a name, skipped");
1948                         continue;
1949                 }
1950                 if(!auth_zones_cfg(az, p)) {
1951                         log_err("cannot config auth zone %s", p->name);
1952                         return 0;
1953                 }
1954         }
1955         az_delete_deleted_zones(az);
1956         if(!auth_zones_read_zones(az))
1957                 return 0;
1958         if(setup) {
1959                 if(!auth_zones_setup_zones(az))
1960                         return 0;
1961         }
1962         return 1;
1963 }
1964
1965 /** delete chunks
1966  * @param at: transfer structure with chunks list.  The chunks and their
1967  *      data are freed.
1968  */
1969 static void
1970 auth_chunks_delete(struct auth_transfer* at)
1971 {
1972         if(at->chunks_first) {
1973                 struct auth_chunk* c, *cn;
1974                 c = at->chunks_first;
1975                 while(c) {
1976                         cn = c->next;
1977                         free(c->data);
1978                         free(c);
1979                         c = cn;
1980                 }
1981         }
1982         at->chunks_first = NULL;
1983         at->chunks_last = NULL;
1984 }
1985
1986 /** free master addr list */
1987 static void
1988 auth_free_master_addrs(struct auth_addr* list)
1989 {
1990         struct auth_addr *n;
1991         while(list) {
1992                 n = list->next;
1993                 free(list);
1994                 list = n;
1995         }
1996 }
1997
1998 /** free the masters list */
1999 static void
2000 auth_free_masters(struct auth_master* list)
2001 {
2002         struct auth_master* n;
2003         while(list) {
2004                 n = list->next;
2005                 auth_free_master_addrs(list->list);
2006                 free(list->host);
2007                 free(list->file);
2008                 free(list);
2009                 list = n;
2010         }
2011 }
2012
2013 /** delete auth xfer structure
2014  * @param xfr: delete this xfer and its tasks.
2015  */
2016 static void
2017 auth_xfer_delete(struct auth_xfer* xfr)
2018 {
2019         if(!xfr) return;
2020         lock_basic_destroy(&xfr->lock);
2021         free(xfr->name);
2022         if(xfr->task_nextprobe) {
2023                 comm_timer_delete(xfr->task_nextprobe->timer);
2024                 free(xfr->task_nextprobe);
2025         }
2026         if(xfr->task_probe) {
2027                 auth_free_masters(xfr->task_probe->masters);
2028                 comm_point_delete(xfr->task_probe->cp);
2029                 free(xfr->task_probe);
2030         }
2031         if(xfr->task_transfer) {
2032                 auth_free_masters(xfr->task_transfer->masters);
2033                 comm_point_delete(xfr->task_transfer->cp);
2034                 if(xfr->task_transfer->chunks_first) {
2035                         auth_chunks_delete(xfr->task_transfer);
2036                 }
2037                 free(xfr->task_transfer);
2038         }
2039         auth_free_masters(xfr->allow_notify_list);
2040         free(xfr);
2041 }
2042
2043 /** helper traverse to delete zones */
2044 static void
2045 auth_zone_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2046 {
2047         struct auth_zone* z = (struct auth_zone*)n->key;
2048         auth_zone_delete(z);
2049 }
2050
2051 /** helper traverse to delete xfer zones */
2052 static void
2053 auth_xfer_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2054 {
2055         struct auth_xfer* z = (struct auth_xfer*)n->key;
2056         auth_xfer_delete(z);
2057 }
2058
2059 void auth_zones_delete(struct auth_zones* az)
2060 {
2061         if(!az) return;
2062         lock_rw_destroy(&az->lock);
2063         traverse_postorder(&az->ztree, auth_zone_del, NULL);
2064         traverse_postorder(&az->xtree, auth_xfer_del, NULL);
2065         free(az);
2066 }
2067
2068 /** true if domain has only nsec3 */
2069 static int
2070 domain_has_only_nsec3(struct auth_data* n)
2071 {
2072         struct auth_rrset* rrset = n->rrsets;
2073         int nsec3_seen = 0;
2074         while(rrset) {
2075                 if(rrset->type == LDNS_RR_TYPE_NSEC3) {
2076                         nsec3_seen = 1;
2077                 } else if(rrset->type != LDNS_RR_TYPE_RRSIG) {
2078                         return 0;
2079                 }
2080                 rrset = rrset->next;
2081         }
2082         return nsec3_seen;
2083 }
2084
2085 /** see if the domain has a wildcard child '*.domain' */
2086 static struct auth_data*
2087 az_find_wildcard_domain(struct auth_zone* z, uint8_t* nm, size_t nmlen)
2088 {
2089         uint8_t wc[LDNS_MAX_DOMAINLEN];
2090         if(nmlen+2 > sizeof(wc))
2091                 return NULL; /* result would be too long */
2092         wc[0] = 1; /* length of wildcard label */
2093         wc[1] = (uint8_t)'*'; /* wildcard label */
2094         memmove(wc+2, nm, nmlen);
2095         return az_find_name(z, wc, nmlen+2);
2096 }
2097
2098 /** find wildcard between qname and cename */
2099 static struct auth_data*
2100 az_find_wildcard(struct auth_zone* z, struct query_info* qinfo,
2101         struct auth_data* ce)
2102 {
2103         uint8_t* nm = qinfo->qname;
2104         size_t nmlen = qinfo->qname_len;
2105         struct auth_data* node;
2106         if(!dname_subdomain_c(nm, z->name))
2107                 return NULL; /* out of zone */
2108         while((node=az_find_wildcard_domain(z, nm, nmlen))==NULL) {
2109                 /* see if we can go up to find the wildcard */
2110                 if(nmlen == z->namelen)
2111                         return NULL; /* top of zone reached */
2112                 if(ce && nmlen == ce->namelen)
2113                         return NULL; /* ce reached */
2114                 if(dname_is_root(nm))
2115                         return NULL; /* cannot go up */
2116                 dname_remove_label(&nm, &nmlen);
2117         }
2118         return node;
2119 }
2120
2121 /** domain is not exact, find first candidate ce (name that matches
2122  * a part of qname) in tree */
2123 static struct auth_data*
2124 az_find_candidate_ce(struct auth_zone* z, struct query_info* qinfo,
2125         struct auth_data* n)
2126 {
2127         uint8_t* nm;
2128         size_t nmlen;
2129         if(n) {
2130                 nm = dname_get_shared_topdomain(qinfo->qname, n->name);
2131         } else {
2132                 nm = qinfo->qname;
2133         }
2134         dname_count_size_labels(nm, &nmlen);
2135         n = az_find_name(z, nm, nmlen);
2136         /* delete labels and go up on name */
2137         while(!n) {
2138                 if(dname_is_root(nm))
2139                         return NULL; /* cannot go up */
2140                 dname_remove_label(&nm, &nmlen);
2141                 n = az_find_name(z, nm, nmlen);
2142         }
2143         return n;
2144 }
2145
2146 /** go up the auth tree to next existing name. */
2147 static struct auth_data*
2148 az_domain_go_up(struct auth_zone* z, struct auth_data* n)
2149 {
2150         uint8_t* nm = n->name;
2151         size_t nmlen = n->namelen;
2152         while(!dname_is_root(nm)) {
2153                 dname_remove_label(&nm, &nmlen);
2154                 if((n=az_find_name(z, nm, nmlen)) != NULL)
2155                         return n;
2156         }
2157         return NULL;
2158 }
2159
2160 /** Find the closest encloser, an name that exists and is above the
2161  * qname.
2162  * return true if the node (param node) is existing, nonobscured and
2163  *      can be used to generate answers from.  It is then also node_exact.
2164  * returns false if the node is not good enough (or it wasn't node_exact)
2165  *      in this case the ce can be filled.
2166  *      if ce is NULL, no ce exists, and likely the zone is completely empty,
2167  *      not even with a zone apex.
2168  *      if ce is nonNULL it is the closest enclosing upper name (that exists
2169  *      itself for answer purposes).  That name may have DNAME, NS or wildcard
2170  *      rrset is the closest DNAME or NS rrset that was found.
2171  */
2172 static int
2173 az_find_ce(struct auth_zone* z, struct query_info* qinfo,
2174         struct auth_data* node, int node_exact, struct auth_data** ce,
2175         struct auth_rrset** rrset)
2176 {
2177         struct auth_data* n = node;
2178         *ce = NULL;
2179         *rrset = NULL;
2180         if(!node_exact) {
2181                 /* if not exact, lookup closest exact match */
2182                 n = az_find_candidate_ce(z, qinfo, n);
2183         } else {
2184                 /* if exact, the node itself is the first candidate ce */
2185                 *ce = n;
2186         }
2187
2188         /* no direct answer from nsec3-only domains */
2189         if(n && domain_has_only_nsec3(n)) {
2190                 node_exact = 0;
2191                 *ce = NULL;
2192         }
2193
2194         /* with exact matches, walk up the labels until we find the
2195          * delegation, or DNAME or zone end */
2196         while(n) {
2197                 /* see if the current candidate has issues */
2198                 /* not zone apex and has type NS */
2199                 if(n->namelen != z->namelen &&
2200                         (*rrset=az_domain_rrset(n, LDNS_RR_TYPE_NS)) &&
2201                         /* delegate here, but DS at exact the dp has notype */
2202                         (qinfo->qtype != LDNS_RR_TYPE_DS ||
2203                         n->namelen != qinfo->qname_len)) {
2204                         /* referral */
2205                         /* this is ce and the lowernode is nonexisting */
2206                         *ce = n;
2207                         return 0;
2208                 }
2209                 /* not equal to qname and has type DNAME */
2210                 if(n->namelen != qinfo->qname_len &&
2211                         (*rrset=az_domain_rrset(n, LDNS_RR_TYPE_DNAME))) {
2212                         /* this is ce and the lowernode is nonexisting */
2213                         *ce = n;
2214                         return 0;
2215                 }
2216
2217                 if(*ce == NULL && !domain_has_only_nsec3(n)) {
2218                         /* if not found yet, this exact name must be
2219                          * our lowest match (but not nsec3onlydomain) */
2220                         *ce = n;
2221                 }
2222
2223                 /* walk up the tree by removing labels from name and lookup */
2224                 n = az_domain_go_up(z, n);
2225         }
2226         /* found no problems, if it was an exact node, it is fine to use */
2227         return node_exact;
2228 }
2229
2230 /** add additional A/AAAA from domain names in rrset rdata (+offset)
2231  * offset is number of bytes in rdata where the dname is located. */
2232 static int
2233 az_add_additionals_from(struct auth_zone* z, struct regional* region,
2234         struct dns_msg* msg, struct auth_rrset* rrset, size_t offset)
2235 {
2236         struct packed_rrset_data* d = rrset->data;
2237         size_t i;
2238         if(!d) return 0;
2239         for(i=0; i<d->count; i++) {
2240                 size_t dlen;
2241                 struct auth_data* domain;
2242                 struct auth_rrset* ref;
2243                 if(d->rr_len[i] < 2+offset)
2244                         continue; /* too short */
2245                 if(!(dlen = dname_valid(d->rr_data[i]+2+offset,
2246                         d->rr_len[i]-2-offset)))
2247                         continue; /* malformed */
2248                 domain = az_find_name(z, d->rr_data[i]+2+offset, dlen);
2249                 if(!domain)
2250                         continue;
2251                 if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_A)) != NULL) {
2252                         if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2253                                 return 0;
2254                 }
2255                 if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_AAAA)) != NULL) {
2256                         if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2257                                 return 0;
2258                 }
2259         }
2260         return 1;
2261 }
2262
2263 /** add negative SOA record (with negative TTL) */
2264 static int
2265 az_add_negative_soa(struct auth_zone* z, struct regional* region,
2266         struct dns_msg* msg)
2267 {
2268         uint32_t minimum;
2269         struct packed_rrset_data* d;
2270         struct auth_rrset* soa;
2271         struct auth_data* apex = az_find_name(z, z->name, z->namelen);
2272         if(!apex) return 0;
2273         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
2274         if(!soa) return 0;
2275         /* must be first to put in message; we want to fix the TTL with
2276          * one RRset here, otherwise we'd need to loop over the RRs to get
2277          * the resulting lower TTL */
2278         log_assert(msg->rep->rrset_count == 0);
2279         if(!msg_add_rrset_ns(z, region, msg, apex, soa)) return 0;
2280         /* fixup TTL */
2281         d = (struct packed_rrset_data*)msg->rep->rrsets[msg->rep->rrset_count-1]->entry.data;
2282         /* last 4 bytes are minimum ttl in network format */
2283         if(d->count == 0) return 0;
2284         if(d->rr_len[0] < 2+4) return 0;
2285         minimum = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-4));
2286         d->ttl = (time_t)minimum;
2287         d->rr_ttl[0] = (time_t)minimum;
2288         msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
2289         msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
2290         msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
2291         return 1;
2292 }
2293
2294 /** See if the query goes to empty nonterminal (that has no auth_data,
2295  * but there are nodes underneath.  We already checked that there are
2296  * not NS, or DNAME above, so that we only need to check if some node
2297  * exists below (with nonempty rr list), return true if emptynonterminal */
2298 static int
2299 az_empty_nonterminal(struct auth_zone* z, struct query_info* qinfo,
2300         struct auth_data* node)
2301 {
2302         struct auth_data* next;
2303         if(!node) {
2304                 /* no smaller was found, use first (smallest) node as the
2305                  * next one */
2306                 next = (struct auth_data*)rbtree_first(&z->data);
2307         } else {
2308                 next = (struct auth_data*)rbtree_next(&node->node);
2309         }
2310         while(next && (rbnode_type*)next != RBTREE_NULL && next->rrsets == NULL) {
2311                 /* the next name has empty rrsets, is an empty nonterminal
2312                  * itself, see if there exists something below it */
2313                 next = (struct auth_data*)rbtree_next(&node->node);
2314         }
2315         if((rbnode_type*)next == RBTREE_NULL || !next) {
2316                 /* there is no next node, so something below it cannot
2317                  * exist */
2318                 return 0;
2319         }
2320         /* a next node exists, if there was something below the query,
2321          * this node has to be it.  See if it is below the query name */
2322         if(dname_strict_subdomain_c(next->name, qinfo->qname))
2323                 return 1;
2324         return 0;
2325 }
2326
2327 /** create synth cname target name in buffer, or fail if too long */
2328 static size_t
2329 synth_cname_buf(uint8_t* qname, size_t qname_len, size_t dname_len,
2330         uint8_t* dtarg, size_t dtarglen, uint8_t* buf, size_t buflen)
2331 {
2332         size_t newlen = qname_len + dtarglen - dname_len;
2333         if(newlen > buflen) {
2334                 /* YXDOMAIN error */
2335                 return 0;
2336         }
2337         /* new name is concatenation of qname front (without DNAME owner)
2338          * and DNAME target name */
2339         memcpy(buf, qname, qname_len-dname_len);
2340         memmove(buf+(qname_len-dname_len), dtarg, dtarglen);
2341         return newlen;
2342 }
2343
2344 /** create synthetic CNAME rrset for in a DNAME answer in region,
2345  * false on alloc failure, cname==NULL when name too long. */
2346 static int
2347 create_synth_cname(uint8_t* qname, size_t qname_len, struct regional* region,
2348         struct auth_data* node, struct auth_rrset* dname, uint16_t dclass,
2349         struct ub_packed_rrset_key** cname)
2350 {
2351         uint8_t buf[LDNS_MAX_DOMAINLEN];
2352         uint8_t* dtarg;
2353         size_t dtarglen, newlen;
2354         struct packed_rrset_data* d;
2355
2356         /* get DNAME target name */
2357         if(dname->data->count < 1) return 0;
2358         if(dname->data->rr_len[0] < 3) return 0; /* at least rdatalen +1 */
2359         dtarg = dname->data->rr_data[0]+2;
2360         dtarglen = dname->data->rr_len[0]-2;
2361         if(sldns_read_uint16(dname->data->rr_data[0]) != dtarglen)
2362                 return 0; /* rdatalen in DNAME rdata is malformed */
2363         if(dname_valid(dtarg, dtarglen) != dtarglen)
2364                 return 0; /* DNAME RR has malformed rdata */
2365
2366         /* synthesize a CNAME */
2367         newlen = synth_cname_buf(qname, qname_len, node->namelen,
2368                 dtarg, dtarglen, buf, sizeof(buf));
2369         if(newlen == 0) {
2370                 /* YXDOMAIN error */
2371                 *cname = NULL;
2372                 return 1;
2373         }
2374         *cname = (struct ub_packed_rrset_key*)regional_alloc(region,
2375                 sizeof(struct ub_packed_rrset_key));
2376         if(!*cname)
2377                 return 0; /* out of memory */
2378         memset(&(*cname)->entry, 0, sizeof((*cname)->entry));
2379         (*cname)->entry.key = (*cname);
2380         (*cname)->rk.type = htons(LDNS_RR_TYPE_CNAME);
2381         (*cname)->rk.rrset_class = htons(dclass);
2382         (*cname)->rk.flags = 0;
2383         (*cname)->rk.dname = regional_alloc_init(region, qname, qname_len);
2384         if(!(*cname)->rk.dname)
2385                 return 0; /* out of memory */
2386         (*cname)->rk.dname_len = qname_len;
2387         (*cname)->entry.hash = rrset_key_hash(&(*cname)->rk);
2388         d = (struct packed_rrset_data*)regional_alloc_zero(region,
2389                 sizeof(struct packed_rrset_data) + sizeof(size_t) +
2390                 sizeof(uint8_t*) + sizeof(time_t) + sizeof(uint16_t)
2391                 + newlen);
2392         if(!d)
2393                 return 0; /* out of memory */
2394         (*cname)->entry.data = d;
2395         d->ttl = 0; /* 0 for synthesized CNAME TTL */
2396         d->count = 1;
2397         d->rrsig_count = 0;
2398         d->trust = rrset_trust_ans_noAA;
2399         d->rr_len = (size_t*)((uint8_t*)d +
2400                 sizeof(struct packed_rrset_data));
2401         d->rr_len[0] = newlen + sizeof(uint16_t);
2402         packed_rrset_ptr_fixup(d);
2403         d->rr_ttl[0] = d->ttl;
2404         sldns_write_uint16(d->rr_data[0], newlen);
2405         memmove(d->rr_data[0] + sizeof(uint16_t), buf, newlen);
2406         return 1;
2407 }
2408
2409 /** add a synthesized CNAME to the answer section */
2410 static int
2411 add_synth_cname(struct auth_zone* z, uint8_t* qname, size_t qname_len,
2412         struct regional* region, struct dns_msg* msg, struct auth_data* dname,
2413         struct auth_rrset* rrset)
2414 {
2415         struct ub_packed_rrset_key* cname;
2416         /* synthesize a CNAME */
2417         if(!create_synth_cname(qname, qname_len, region, dname, rrset,
2418                 z->dclass, &cname)) {
2419                 /* out of memory */
2420                 return 0;
2421         }
2422         if(!cname) {
2423                 /* cname cannot be create because of YXDOMAIN */
2424                 msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
2425                 return 1;
2426         }
2427         /* add cname to message */
2428         if(!msg_grow_array(region, msg))
2429                 return 0;
2430         msg->rep->rrsets[msg->rep->rrset_count] = cname;
2431         msg->rep->rrset_count++;
2432         msg->rep->an_numrrsets++;
2433         msg_ttl(msg);
2434         return 1;
2435 }
2436
2437 /** Change a dname to a different one, for wildcard namechange */
2438 static void
2439 az_change_dnames(struct dns_msg* msg, uint8_t* oldname, uint8_t* newname,
2440         size_t newlen, int an_only)
2441 {
2442         size_t i;
2443         size_t start = 0, end = msg->rep->rrset_count;
2444         if(!an_only) start = msg->rep->an_numrrsets;
2445         if(an_only) end = msg->rep->an_numrrsets;
2446         for(i=start; i<end; i++) {
2447                 /* allocated in region so we can change the ptrs */
2448                 if(query_dname_compare(msg->rep->rrsets[i]->rk.dname, oldname)
2449                         == 0) {
2450                         msg->rep->rrsets[i]->rk.dname = newname;
2451                         msg->rep->rrsets[i]->rk.dname_len = newlen;
2452                 }
2453         }
2454 }
2455
2456 /** find NSEC record covering the query */
2457 static struct auth_rrset*
2458 az_find_nsec_cover(struct auth_zone* z, struct auth_data** node)
2459 {
2460         uint8_t* nm = (*node)->name;
2461         size_t nmlen = (*node)->namelen;
2462         struct auth_rrset* rrset;
2463         /* find the NSEC for the smallest-or-equal node */
2464         /* if node == NULL, we did not find a smaller name.  But the zone
2465          * name is the smallest name and should have an NSEC. So there is
2466          * no NSEC to return (for a properly signed zone) */
2467         /* for empty nonterminals, the auth-data node should not exist,
2468          * and thus we don't need to go rbtree_previous here to find
2469          * a domain with an NSEC record */
2470         /* but there could be glue, and if this is node, then it has no NSEC.
2471          * Go up to find nonglue (previous) NSEC-holding nodes */
2472         while((rrset=az_domain_rrset(*node, LDNS_RR_TYPE_NSEC)) == NULL) {
2473                 if(dname_is_root(nm)) return NULL;
2474                 if(nmlen == z->namelen) return NULL;
2475                 dname_remove_label(&nm, &nmlen);
2476                 /* adjust *node for the nsec rrset to find in */
2477                 *node = az_find_name(z, nm, nmlen);
2478         }
2479         return rrset;
2480 }
2481
2482 /** Find NSEC and add for wildcard denial */
2483 static int
2484 az_nsec_wildcard_denial(struct auth_zone* z, struct regional* region,
2485         struct dns_msg* msg, uint8_t* cenm, size_t cenmlen)
2486 {
2487         struct query_info qinfo;
2488         int node_exact;
2489         struct auth_data* node;
2490         struct auth_rrset* nsec;
2491         uint8_t wc[LDNS_MAX_DOMAINLEN];
2492         if(cenmlen+2 > sizeof(wc))
2493                 return 0; /* result would be too long */
2494         wc[0] = 1; /* length of wildcard label */
2495         wc[1] = (uint8_t)'*'; /* wildcard label */
2496         memmove(wc+2, cenm, cenmlen);
2497
2498         /* we have '*.ce' in wc wildcard name buffer */
2499         /* get nsec cover for that */
2500         qinfo.qname = wc;
2501         qinfo.qname_len = cenmlen+2;
2502         qinfo.qtype = 0;
2503         qinfo.qclass = 0;
2504         az_find_domain(z, &qinfo, &node_exact, &node);
2505         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
2506                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
2507         }
2508         return 1;
2509 }
2510
2511 /** Find the NSEC3PARAM rrset (if any) and if true you have the parameters */
2512 static int
2513 az_nsec3_param(struct auth_zone* z, int* algo, size_t* iter, uint8_t** salt,
2514         size_t* saltlen)
2515 {
2516         struct auth_data* apex;
2517         struct auth_rrset* param;
2518         size_t i;
2519         apex = az_find_name(z, z->name, z->namelen);
2520         if(!apex) return 0;
2521         param = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC3PARAM);
2522         if(!param || param->data->count==0)
2523                 return 0; /* no RRset or no RRs in rrset */
2524         /* find out which NSEC3PARAM RR has supported parameters */
2525         /* skip unknown flags (dynamic signer is recalculating nsec3 chain) */
2526         for(i=0; i<param->data->count; i++) {
2527                 uint8_t* rdata = param->data->rr_data[i]+2;
2528                 size_t rdatalen = param->data->rr_len[i];
2529                 if(rdatalen < 2+5)
2530                         continue; /* too short */
2531                 if(!nsec3_hash_algo_size_supported((int)(rdata[0])))
2532                         continue; /* unsupported algo */
2533                 if(rdatalen < (size_t)(2+5+(size_t)rdata[4]))
2534                         continue; /* salt missing */
2535                 if((rdata[1]&NSEC3_UNKNOWN_FLAGS)!=0)
2536                         continue; /* unknown flags */
2537                 *algo = (int)(rdata[0]);
2538                 *iter = sldns_read_uint16(rdata+2);
2539                 *saltlen = rdata[4];
2540                 if(*saltlen == 0)
2541                         *salt = NULL;
2542                 else    *salt = rdata+5;
2543                 return 1;
2544         }
2545         /* no supported params */
2546         return 0;
2547 }
2548
2549 /** Hash a name with nsec3param into buffer, it has zone name appended.
2550  * return length of hash */
2551 static size_t
2552 az_nsec3_hash(uint8_t* buf, size_t buflen, uint8_t* nm, size_t nmlen,
2553         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2554 {
2555         size_t hlen = nsec3_hash_algo_size_supported(algo);
2556         /* buffer has domain name, nsec3hash, and 256 is for max saltlen
2557          * (salt has 0-255 length) */
2558         unsigned char p[LDNS_MAX_DOMAINLEN+1+N3HASHBUFLEN+256];
2559         size_t i;
2560         if(nmlen+saltlen > sizeof(p) || hlen+saltlen > sizeof(p))
2561                 return 0;
2562         if(hlen > buflen)
2563                 return 0; /* somehow too large for destination buffer */
2564         /* hashfunc(name, salt) */
2565         memmove(p, nm, nmlen);
2566         query_dname_tolower(p);
2567         memmove(p+nmlen, salt, saltlen);
2568         (void)secalgo_nsec3_hash(algo, p, nmlen+saltlen, (unsigned char*)buf);
2569         for(i=0; i<iter; i++) {
2570                 /* hashfunc(hash, salt) */
2571                 memmove(p, buf, hlen);
2572                 memmove(p+hlen, salt, saltlen);
2573                 (void)secalgo_nsec3_hash(algo, p, hlen+saltlen,
2574                         (unsigned char*)buf);
2575         }
2576         return hlen;
2577 }
2578
2579 /** Hash name and return b32encoded hashname for lookup, zone name appended */
2580 static int
2581 az_nsec3_hashname(struct auth_zone* z, uint8_t* hashname, size_t* hashnmlen,
2582         uint8_t* nm, size_t nmlen, int algo, size_t iter, uint8_t* salt,
2583         size_t saltlen)
2584 {
2585         uint8_t hash[N3HASHBUFLEN];
2586         size_t hlen;
2587         int ret;
2588         hlen = az_nsec3_hash(hash, sizeof(hash), nm, nmlen, algo, iter,
2589                 salt, saltlen);
2590         if(!hlen) return 0;
2591         /* b32 encode */
2592         if(*hashnmlen < hlen*2+1+z->namelen) /* approx b32 as hexb16 */
2593                 return 0;
2594         ret = sldns_b32_ntop_extended_hex(hash, hlen, (char*)(hashname+1),
2595                 (*hashnmlen)-1);
2596         if(ret<1)
2597                 return 0;
2598         hashname[0] = (uint8_t)ret;
2599         ret++;
2600         if((*hashnmlen) - ret < z->namelen)
2601                 return 0;
2602         memmove(hashname+ret, z->name, z->namelen);
2603         *hashnmlen = z->namelen+(size_t)ret;
2604         return 1;
2605 }
2606
2607 /** Find the datanode that covers the nsec3hash-name */
2608 static struct auth_data*
2609 az_nsec3_findnode(struct auth_zone* z, uint8_t* hashnm, size_t hashnmlen)
2610 {
2611         struct query_info qinfo;
2612         struct auth_data* node;
2613         int node_exact;
2614         qinfo.qclass = 0;
2615         qinfo.qtype = 0;
2616         qinfo.qname = hashnm;
2617         qinfo.qname_len = hashnmlen;
2618         /* because canonical ordering and b32 nsec3 ordering are the same.
2619          * this is a good lookup to find the nsec3 name. */
2620         az_find_domain(z, &qinfo, &node_exact, &node);
2621         /* but we may have to skip non-nsec3 nodes */
2622         /* this may be a lot, the way to speed that up is to have a
2623          * separate nsec3 tree with nsec3 nodes */
2624         while(node && (rbnode_type*)node != RBTREE_NULL &&
2625                 !az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2626                 node = (struct auth_data*)rbtree_previous(&node->node);
2627         }
2628         if((rbnode_type*)node == RBTREE_NULL)
2629                 node = NULL;
2630         return node;
2631 }
2632
2633 /** Find cover for hashed(nm, nmlen) (or NULL) */
2634 static struct auth_data*
2635 az_nsec3_find_cover(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2636         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2637 {
2638         struct auth_data* node;
2639         uint8_t hname[LDNS_MAX_DOMAINLEN];
2640         size_t hlen = sizeof(hname);
2641         if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2642                 salt, saltlen))
2643                 return NULL;
2644         node = az_nsec3_findnode(z, hname, hlen);
2645         if(node)
2646                 return node;
2647         /* we did not find any, perhaps because the NSEC3 hash is before
2648          * the first hash, we have to find the 'last hash' in the zone */
2649         node = (struct auth_data*)rbtree_last(&z->data);
2650         while(node && (rbnode_type*)node != RBTREE_NULL &&
2651                 !az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2652                 node = (struct auth_data*)rbtree_previous(&node->node);
2653         }
2654         if((rbnode_type*)node == RBTREE_NULL)
2655                 node = NULL;
2656         return node;
2657 }
2658
2659 /** Find exact match for hashed(nm, nmlen) NSEC3 record or NULL */
2660 static struct auth_data*
2661 az_nsec3_find_exact(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2662         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2663 {
2664         struct auth_data* node;
2665         uint8_t hname[LDNS_MAX_DOMAINLEN];
2666         size_t hlen = sizeof(hname);
2667         if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2668                 salt, saltlen))
2669                 return NULL;
2670         node = az_find_name(z, hname, hlen);
2671         if(az_domain_rrset(node, LDNS_RR_TYPE_NSEC3))
2672                 return node;
2673         return NULL;
2674 }
2675
2676 /** Return nextcloser name (as a ref into the qname).  This is one label
2677  * more than the cenm (cename must be a suffix of qname) */
2678 static void
2679 az_nsec3_get_nextcloser(uint8_t* cenm, uint8_t* qname, size_t qname_len,
2680         uint8_t** nx, size_t* nxlen)
2681 {
2682         int celabs = dname_count_labels(cenm);
2683         int qlabs = dname_count_labels(qname);
2684         int strip = qlabs - celabs -1;
2685         log_assert(dname_strict_subdomain(qname, qlabs, cenm, celabs));
2686         *nx = qname;
2687         *nxlen = qname_len;
2688         if(strip>0)
2689                 dname_remove_labels(nx, nxlen, strip);
2690 }
2691
2692 /** Find the closest encloser that has exact NSEC3.
2693  * updated cenm to the new name. If it went up no-exact-ce is true. */
2694 static struct auth_data*
2695 az_nsec3_find_ce(struct auth_zone* z, uint8_t** cenm, size_t* cenmlen,
2696         int* no_exact_ce, int algo, size_t iter, uint8_t* salt, size_t saltlen)
2697 {
2698         struct auth_data* node;
2699         while((node = az_nsec3_find_exact(z, *cenm, *cenmlen,
2700                 algo, iter, salt, saltlen)) == NULL) {
2701                 if(*cenmlen == z->namelen) {
2702                         /* next step up would take us out of the zone. fail */
2703                         return NULL;
2704                 }
2705                 *no_exact_ce = 1;
2706                 dname_remove_label(cenm, cenmlen);
2707         }
2708         return node;
2709 }
2710
2711 /* Insert NSEC3 record in authority section, if NULL does nothing */
2712 static int
2713 az_nsec3_insert(struct auth_zone* z, struct regional* region,
2714         struct dns_msg* msg, struct auth_data* node)
2715 {
2716         struct auth_rrset* nsec3;
2717         if(!node) return 1; /* no node, skip this */
2718         nsec3 = az_domain_rrset(node, LDNS_RR_TYPE_NSEC3);
2719         if(!nsec3) return 1; /* if no nsec3 RR, skip it */
2720         if(!msg_add_rrset_ns(z, region, msg, node, nsec3)) return 0;
2721         return 1;
2722 }
2723
2724 /** add NSEC3 records to the zone for the nsec3 proof.
2725  * Specify with the flags with parts of the proof are required.
2726  * the ce is the exact matching name (for notype) but also delegation points.
2727  * qname is the one where the nextcloser name can be derived from.
2728  * If NSEC3 is not properly there (in the zone) nothing is added.
2729  * always enabled: include nsec3 proving about the Closest Encloser.
2730  *      that is an exact match that should exist for it.
2731  *      If that does not exist, a higher exact match + nxproof is enabled
2732  *      (for some sort of opt-out empty nonterminal cases).
2733  * nxproof: include denial of the qname.
2734  * wcproof: include denial of wildcard (wildcard.ce).
2735  */
2736 static int
2737 az_add_nsec3_proof(struct auth_zone* z, struct regional* region,
2738         struct dns_msg* msg, uint8_t* cenm, size_t cenmlen, uint8_t* qname,
2739         size_t qname_len, int nxproof, int wcproof)
2740 {
2741         int algo;
2742         size_t iter, saltlen;
2743         uint8_t* salt;
2744         int no_exact_ce = 0;
2745         struct auth_data* node;
2746
2747         /* find parameters of nsec3 proof */
2748         if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen))
2749                 return 1; /* no nsec3 */
2750         /* find ce that has an NSEC3 */
2751         node = az_nsec3_find_ce(z, &cenm, &cenmlen, &no_exact_ce,
2752                 algo, iter, salt, saltlen);
2753         if(no_exact_ce) nxproof = 1;
2754         if(!az_nsec3_insert(z, region, msg, node))
2755                 return 0;
2756
2757         if(nxproof) {
2758                 uint8_t* nx;
2759                 size_t nxlen;
2760                 /* create nextcloser domain name */
2761                 az_nsec3_get_nextcloser(cenm, qname, qname_len, &nx, &nxlen);
2762                 /* find nsec3 that matches or covers it */
2763                 node = az_nsec3_find_cover(z, nx, nxlen, algo, iter, salt,
2764                         saltlen);
2765                 if(!az_nsec3_insert(z, region, msg, node))
2766                         return 0;
2767         }
2768         if(wcproof) {
2769                 /* create wildcard name *.ce */
2770                 uint8_t wc[LDNS_MAX_DOMAINLEN];
2771                 size_t wclen;
2772                 if(cenmlen+2 > sizeof(wc))
2773                         return 0; /* result would be too long */
2774                 wc[0] = 1; /* length of wildcard label */
2775                 wc[1] = (uint8_t)'*'; /* wildcard label */
2776                 memmove(wc+2, cenm, cenmlen);
2777                 wclen = cenmlen+2;
2778                 /* find nsec3 that matches or covers it */
2779                 node = az_nsec3_find_cover(z, wc, wclen, algo, iter, salt,
2780                         saltlen);
2781                 if(!az_nsec3_insert(z, region, msg, node))
2782                         return 0;
2783         }
2784         return 1;
2785 }
2786
2787 /** generate answer for positive answer */
2788 static int
2789 az_generate_positive_answer(struct auth_zone* z, struct regional* region,
2790         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
2791 {
2792         if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2793         /* see if we want additional rrs */
2794         if(rrset->type == LDNS_RR_TYPE_MX) {
2795                 if(!az_add_additionals_from(z, region, msg, rrset, 2))
2796                         return 0;
2797         } else if(rrset->type == LDNS_RR_TYPE_SRV) {
2798                 if(!az_add_additionals_from(z, region, msg, rrset, 6))
2799                         return 0;
2800         } else if(rrset->type == LDNS_RR_TYPE_NS) {
2801                 if(!az_add_additionals_from(z, region, msg, rrset, 0))
2802                         return 0;
2803         }
2804         return 1;
2805 }
2806
2807 /** generate answer for type ANY answer */
2808 static int
2809 az_generate_any_answer(struct auth_zone* z, struct regional* region,
2810         struct dns_msg* msg, struct auth_data* node)
2811 {
2812         struct auth_rrset* rrset;
2813         int added = 0;
2814         /* add a couple (at least one) RRs */
2815         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_SOA)) != NULL) {
2816                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2817                 added++;
2818         }
2819         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_MX)) != NULL) {
2820                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2821                 added++;
2822         }
2823         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_A)) != NULL) {
2824                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2825                 added++;
2826         }
2827         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_AAAA)) != NULL) {
2828                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2829                 added++;
2830         }
2831         if(added == 0 && node->rrsets) {
2832                 if(!msg_add_rrset_an(z, region, msg, node,
2833                         node->rrsets)) return 0;
2834         }
2835         return 1;
2836 }
2837
2838 /** follow cname chain and add more data to the answer section */
2839 static int
2840 follow_cname_chain(struct auth_zone* z, uint16_t qtype,
2841         struct regional* region, struct dns_msg* msg,
2842         struct packed_rrset_data* d)
2843 {
2844         int maxchain = 0;
2845         /* see if we can add the target of the CNAME into the answer */
2846         while(maxchain++ < MAX_CNAME_CHAIN) {
2847                 struct auth_data* node;
2848                 struct auth_rrset* rrset;
2849                 size_t clen;
2850                 /* d has cname rdata */
2851                 if(d->count == 0) break; /* no CNAME */
2852                 if(d->rr_len[0] < 2+1) break; /* too small */
2853                 if((clen=dname_valid(d->rr_data[0]+2, d->rr_len[0]-2))==0)
2854                         break; /* malformed */
2855                 if(!dname_subdomain_c(d->rr_data[0]+2, z->name))
2856                         break; /* target out of zone */
2857                 if((node = az_find_name(z, d->rr_data[0]+2, clen))==NULL)
2858                         break; /* no such target name */
2859                 if((rrset=az_domain_rrset(node, qtype))!=NULL) {
2860                         /* done we found the target */
2861                         if(!msg_add_rrset_an(z, region, msg, node, rrset))
2862                                 return 0;
2863                         break;
2864                 }
2865                 if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME))==NULL)
2866                         break; /* no further CNAME chain, notype */
2867                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2868                 d = rrset->data;
2869         }
2870         return 1;
2871 }
2872
2873 /** generate answer for cname answer */
2874 static int
2875 az_generate_cname_answer(struct auth_zone* z, struct query_info* qinfo,
2876         struct regional* region, struct dns_msg* msg,
2877         struct auth_data* node, struct auth_rrset* rrset)
2878 {
2879         if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
2880         if(!rrset) return 1;
2881         if(!follow_cname_chain(z, qinfo->qtype, region, msg, rrset->data))
2882                 return 0;
2883         return 1;
2884 }
2885
2886 /** generate answer for notype answer */
2887 static int
2888 az_generate_notype_answer(struct auth_zone* z, struct regional* region,
2889         struct dns_msg* msg, struct auth_data* node)
2890 {
2891         struct auth_rrset* rrset;
2892         if(!az_add_negative_soa(z, region, msg)) return 0;
2893         /* DNSSEC denial NSEC */
2894         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_NSEC))!=NULL) {
2895                 if(!msg_add_rrset_ns(z, region, msg, node, rrset)) return 0;
2896         } else if(node) {
2897                 /* DNSSEC denial NSEC3 */
2898                 if(!az_add_nsec3_proof(z, region, msg, node->name,
2899                         node->namelen, msg->qinfo.qname,
2900                         msg->qinfo.qname_len, 0, 0))
2901                         return 0;
2902         }
2903         return 1;
2904 }
2905
2906 /** generate answer for referral answer */
2907 static int
2908 az_generate_referral_answer(struct auth_zone* z, struct regional* region,
2909         struct dns_msg* msg, struct auth_data* ce, struct auth_rrset* rrset)
2910 {
2911         struct auth_rrset* ds, *nsec;
2912         /* turn off AA flag, referral is nonAA because it leaves the zone */
2913         log_assert(ce);
2914         msg->rep->flags &= ~BIT_AA;
2915         if(!msg_add_rrset_ns(z, region, msg, ce, rrset)) return 0;
2916         /* add DS or deny it */
2917         if((ds=az_domain_rrset(ce, LDNS_RR_TYPE_DS))!=NULL) {
2918                 if(!msg_add_rrset_ns(z, region, msg, ce, ds)) return 0;
2919         } else {
2920                 /* deny the DS */
2921                 if((nsec=az_domain_rrset(ce, LDNS_RR_TYPE_NSEC))!=NULL) {
2922                         if(!msg_add_rrset_ns(z, region, msg, ce, nsec))
2923                                 return 0;
2924                 } else {
2925                         if(!az_add_nsec3_proof(z, region, msg, ce->name,
2926                                 ce->namelen, msg->qinfo.qname,
2927                                 msg->qinfo.qname_len, 0, 0))
2928                                 return 0;
2929                 }
2930         }
2931         /* add additional rrs for type NS */
2932         if(!az_add_additionals_from(z, region, msg, rrset, 0)) return 0;
2933         return 1;
2934 }
2935
2936 /** generate answer for DNAME answer */
2937 static int
2938 az_generate_dname_answer(struct auth_zone* z, struct query_info* qinfo,
2939         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
2940         struct auth_rrset* rrset)
2941 {
2942         log_assert(ce);
2943         /* add the DNAME and then a CNAME */
2944         if(!msg_add_rrset_an(z, region, msg, ce, rrset)) return 0;
2945         if(!add_synth_cname(z, qinfo->qname, qinfo->qname_len, region,
2946                 msg, ce, rrset)) return 0;
2947         if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_YXDOMAIN)
2948                 return 1;
2949         if(msg->rep->rrset_count == 0 ||
2950                 !msg->rep->rrsets[msg->rep->rrset_count-1])
2951                 return 0;
2952         if(!follow_cname_chain(z, qinfo->qtype, region, msg,
2953                 (struct packed_rrset_data*)msg->rep->rrsets[
2954                 msg->rep->rrset_count-1]->entry.data))
2955                 return 0;
2956         return 1;
2957 }
2958
2959 /** generate answer for wildcard answer */
2960 static int
2961 az_generate_wildcard_answer(struct auth_zone* z, struct query_info* qinfo,
2962         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
2963         struct auth_data* wildcard, struct auth_data* node)
2964 {
2965         struct auth_rrset* rrset, *nsec;
2966         if((rrset=az_domain_rrset(wildcard, qinfo->qtype)) != NULL) {
2967                 /* wildcard has type, add it */
2968                 if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
2969                         return 0;
2970                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
2971                         msg->qinfo.qname_len, 1);
2972         } else if((rrset=az_domain_rrset(wildcard, LDNS_RR_TYPE_CNAME))!=NULL) {
2973                 /* wildcard has cname instead, do that */
2974                 if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
2975                         return 0;
2976                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
2977                         msg->qinfo.qname_len, 1);
2978                 if(!follow_cname_chain(z, qinfo->qtype, region, msg,
2979                         rrset->data))
2980                         return 0;
2981         } else if(qinfo->qtype == LDNS_RR_TYPE_ANY && wildcard->rrsets) {
2982                 /* add ANY rrsets from wildcard node */
2983                 if(!az_generate_any_answer(z, region, msg, wildcard))
2984                         return 0;
2985                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
2986                         msg->qinfo.qname_len, 1);
2987         } else {
2988                 /* wildcard has nodata, notype answer */
2989                 /* call other notype routine for dnssec notype denials */
2990                 if(!az_generate_notype_answer(z, region, msg, wildcard))
2991                         return 0;
2992         }
2993
2994         /* ce and node for dnssec denial of wildcard original name */
2995         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
2996                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
2997         } else if(ce) {
2998                 if(!az_add_nsec3_proof(z, region, msg, ce->name,
2999                         ce->namelen, msg->qinfo.qname,
3000                         msg->qinfo.qname_len, 1, 0))
3001                         return 0;
3002         }
3003
3004         /* fixup name of wildcard from *.zone to qname, use already allocated
3005          * pointer to msg qname */
3006         az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3007                 msg->qinfo.qname_len, 0);
3008         return 1;
3009 }
3010
3011 /** generate answer for nxdomain answer */
3012 static int
3013 az_generate_nxdomain_answer(struct auth_zone* z, struct regional* region,
3014         struct dns_msg* msg, struct auth_data* ce, struct auth_data* node)
3015 {
3016         struct auth_rrset* nsec;
3017         msg->rep->flags |= LDNS_RCODE_NXDOMAIN;
3018         if(!az_add_negative_soa(z, region, msg)) return 0;
3019         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3020                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3021                 if(ce && !az_nsec_wildcard_denial(z, region, msg, ce->name,
3022                         ce->namelen)) return 0;
3023         } else if(ce) {
3024                 if(!az_add_nsec3_proof(z, region, msg, ce->name,
3025                         ce->namelen, msg->qinfo.qname,
3026                         msg->qinfo.qname_len, 1, 1))
3027                         return 0;
3028         }
3029         return 1;
3030 }
3031
3032 /** Create answers when an exact match exists for the domain name */
3033 static int
3034 az_generate_answer_with_node(struct auth_zone* z, struct query_info* qinfo,
3035         struct regional* region, struct dns_msg* msg, struct auth_data* node)
3036 {
3037         struct auth_rrset* rrset;
3038         /* positive answer, rrset we are looking for exists */
3039         if((rrset=az_domain_rrset(node, qinfo->qtype)) != NULL) {
3040                 return az_generate_positive_answer(z, region, msg, node, rrset);
3041         }
3042         /* CNAME? */
3043         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME)) != NULL) {
3044                 return az_generate_cname_answer(z, qinfo, region, msg,
3045                         node, rrset);
3046         }
3047         /* type ANY ? */
3048         if(qinfo->qtype == LDNS_RR_TYPE_ANY) {
3049                 return az_generate_any_answer(z, region, msg, node);
3050         }
3051         /* NOERROR/NODATA (no such type at domain name) */
3052         return az_generate_notype_answer(z, region, msg, node);
3053 }
3054
3055 /** Generate answer without an existing-node that we can use.
3056  * So it'll be a referral, DNAME or nxdomain */
3057 static int
3058 az_generate_answer_nonexistnode(struct auth_zone* z, struct query_info* qinfo,
3059         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3060         struct auth_rrset* rrset, struct auth_data* node)
3061 {
3062         struct auth_data* wildcard;
3063
3064         /* we do not have an exact matching name (that exists) */
3065         /* see if we have a NS or DNAME in the ce */
3066         if(ce && rrset && rrset->type == LDNS_RR_TYPE_NS) {
3067                 return az_generate_referral_answer(z, region, msg, ce, rrset);
3068         }
3069         if(ce && rrset && rrset->type == LDNS_RR_TYPE_DNAME) {
3070                 return az_generate_dname_answer(z, qinfo, region, msg, ce,
3071                         rrset);
3072         }
3073         /* if there is an empty nonterminal, wildcard and nxdomain don't
3074          * happen, it is a notype answer */
3075         if(az_empty_nonterminal(z, qinfo, node)) {
3076                 return az_generate_notype_answer(z, region, msg, node);
3077         }
3078         /* see if we have a wildcard under the ce */
3079         if((wildcard=az_find_wildcard(z, qinfo, ce)) != NULL) {
3080                 return az_generate_wildcard_answer(z, qinfo, region, msg,
3081                         ce, wildcard, node);
3082         }
3083         /* generate nxdomain answer */
3084         return az_generate_nxdomain_answer(z, region, msg, ce, node);
3085 }
3086
3087 /** Lookup answer in a zone. */
3088 static int
3089 auth_zone_generate_answer(struct auth_zone* z, struct query_info* qinfo,
3090         struct regional* region, struct dns_msg** msg, int* fallback)
3091 {
3092         struct auth_data* node, *ce;
3093         struct auth_rrset* rrset;
3094         int node_exact, node_exists;
3095         /* does the zone want fallback in case of failure? */
3096         *fallback = z->fallback_enabled;
3097         if(!(*msg=msg_create(region, qinfo))) return 0;
3098
3099         /* lookup if there is a matching domain name for the query */
3100         az_find_domain(z, qinfo, &node_exact, &node);
3101
3102         /* see if node exists for generating answers from (i.e. not glue and
3103          * obscured by NS or DNAME or NSEC3-only), and also return the
3104          * closest-encloser from that, closest node that should be used
3105          * to generate answers from that is above the query */
3106         node_exists = az_find_ce(z, qinfo, node, node_exact, &ce, &rrset);
3107
3108         if(verbosity >= VERB_ALGO) {
3109                 char zname[256], qname[256], nname[256], cename[256],
3110                         tpstr[32], rrstr[32];
3111                 sldns_wire2str_dname_buf(qinfo->qname, qinfo->qname_len, qname,
3112                         sizeof(qname));
3113                 sldns_wire2str_type_buf(qinfo->qtype, tpstr, sizeof(tpstr));
3114                 sldns_wire2str_dname_buf(z->name, z->namelen, zname,
3115                         sizeof(zname));
3116                 if(node)
3117                         sldns_wire2str_dname_buf(node->name, node->namelen,
3118                                 nname, sizeof(nname));
3119                 else    snprintf(nname, sizeof(nname), "NULL");
3120                 if(ce)
3121                         sldns_wire2str_dname_buf(ce->name, ce->namelen,
3122                                 cename, sizeof(cename));
3123                 else    snprintf(cename, sizeof(cename), "NULL");
3124                 if(rrset) sldns_wire2str_type_buf(rrset->type, rrstr,
3125                         sizeof(rrstr));
3126                 else    snprintf(rrstr, sizeof(rrstr), "NULL");
3127                 log_info("auth_zone %s query %s %s, domain %s %s %s, "
3128                         "ce %s, rrset %s", zname, qname, tpstr, nname,
3129                         (node_exact?"exact":"notexact"),
3130                         (node_exists?"exist":"notexist"), cename, rrstr);
3131         }
3132
3133         if(node_exists) {
3134                 /* the node is fine, generate answer from node */
3135                 return az_generate_answer_with_node(z, qinfo, region, *msg,
3136                         node);
3137         }
3138         return az_generate_answer_nonexistnode(z, qinfo, region, *msg,
3139                 ce, rrset, node);
3140 }
3141
3142 int auth_zones_lookup(struct auth_zones* az, struct query_info* qinfo,
3143         struct regional* region, struct dns_msg** msg, int* fallback,
3144         uint8_t* dp_nm, size_t dp_nmlen)
3145 {
3146         int r;
3147         struct auth_zone* z;
3148         /* find the zone that should contain the answer. */
3149         lock_rw_rdlock(&az->lock);
3150         z = auth_zone_find(az, dp_nm, dp_nmlen, qinfo->qclass);
3151         if(!z) {
3152                 lock_rw_unlock(&az->lock);
3153                 /* no auth zone, fallback to internet */
3154                 *fallback = 1;
3155                 return 0;
3156         }
3157         lock_rw_rdlock(&z->lock);
3158         lock_rw_unlock(&az->lock);
3159
3160         /* if not for upstream queries, fallback */
3161         if(!z->for_upstream) {
3162                 lock_rw_unlock(&z->lock);
3163                 *fallback = 1;
3164                 return 0;
3165         }
3166         /* see what answer that zone would generate */
3167         r = auth_zone_generate_answer(z, qinfo, region, msg, fallback);
3168         lock_rw_unlock(&z->lock);
3169         return r;
3170 }
3171
3172 /** encode auth answer */
3173 static void
3174 auth_answer_encode(struct query_info* qinfo, struct module_env* env,
3175         struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3176         struct regional* temp, struct dns_msg* msg)
3177 {
3178         uint16_t udpsize;
3179         udpsize = edns->udp_size;
3180         edns->edns_version = EDNS_ADVERTISED_VERSION;
3181         edns->udp_size = EDNS_ADVERTISED_SIZE;
3182         edns->ext_rcode = 0;
3183         edns->bits &= EDNS_DO;
3184
3185         if(!inplace_cb_reply_local_call(env, qinfo, NULL, msg->rep,
3186                 (int)FLAGS_GET_RCODE(msg->rep->flags), edns, repinfo, temp)
3187                 || !reply_info_answer_encode(qinfo, msg->rep,
3188                 *(uint16_t*)sldns_buffer_begin(buf),
3189                 sldns_buffer_read_u16_at(buf, 2),
3190                 buf, 0, 0, temp, udpsize, edns,
3191                 (int)(edns->bits&EDNS_DO), 0)) {
3192                 error_encode(buf, (LDNS_RCODE_SERVFAIL|BIT_AA), qinfo,
3193                         *(uint16_t*)sldns_buffer_begin(buf),
3194                         sldns_buffer_read_u16_at(buf, 2), edns);
3195         }
3196 }
3197
3198 /** encode auth error answer */
3199 static void
3200 auth_error_encode(struct query_info* qinfo, struct module_env* env,
3201         struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3202         struct regional* temp, int rcode)
3203 {
3204         edns->edns_version = EDNS_ADVERTISED_VERSION;
3205         edns->udp_size = EDNS_ADVERTISED_SIZE;
3206         edns->ext_rcode = 0;
3207         edns->bits &= EDNS_DO;
3208
3209         if(!inplace_cb_reply_local_call(env, qinfo, NULL, NULL,
3210                 rcode, edns, repinfo, temp))
3211                 edns->opt_list = NULL;
3212         error_encode(buf, rcode|BIT_AA, qinfo,
3213                 *(uint16_t*)sldns_buffer_begin(buf),
3214                 sldns_buffer_read_u16_at(buf, 2), edns);
3215 }
3216
3217 int auth_zones_answer(struct auth_zones* az, struct module_env* env,
3218         struct query_info* qinfo, struct edns_data* edns,
3219         struct comm_reply* repinfo, struct sldns_buffer* buf, struct regional* temp)
3220 {
3221         struct dns_msg* msg = NULL;
3222         struct auth_zone* z;
3223         int r;
3224         int fallback = 0;
3225
3226         lock_rw_rdlock(&az->lock);
3227         if(!az->have_downstream) {
3228                 /* no downstream auth zones */
3229                 lock_rw_unlock(&az->lock);
3230                 return 0;
3231         }
3232         if(qinfo->qtype == LDNS_RR_TYPE_DS) {
3233                 uint8_t* delname = qinfo->qname;
3234                 size_t delnamelen = qinfo->qname_len;
3235                 dname_remove_label(&delname, &delnamelen);
3236                 z = auth_zones_find_zone(az, delname, delnamelen,
3237                         qinfo->qclass);
3238         } else {
3239                 z = auth_zones_find_zone(az, qinfo->qname, qinfo->qname_len,
3240                         qinfo->qclass);
3241         }
3242         if(!z) {
3243                 /* no zone above it */
3244                 lock_rw_unlock(&az->lock);
3245                 return 0;
3246         }
3247         lock_rw_rdlock(&z->lock);
3248         lock_rw_unlock(&az->lock);
3249         if(!z->for_downstream) {
3250                 lock_rw_unlock(&z->lock);
3251                 return 0;
3252         }
3253
3254         /* answer it from zone z */
3255         r = auth_zone_generate_answer(z, qinfo, temp, &msg, &fallback);
3256         lock_rw_unlock(&z->lock);
3257         if(!r && fallback) {
3258                 /* fallback to regular answering (recursive) */
3259                 return 0;
3260         }
3261         lock_rw_wrlock(&az->lock);
3262         az->num_query_down++;
3263         lock_rw_unlock(&az->lock);
3264
3265         /* encode answer */
3266         if(!r)
3267                 auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3268                         LDNS_RCODE_SERVFAIL);
3269         else    auth_answer_encode(qinfo, env, edns, repinfo, buf, temp, msg);
3270
3271         return 1;
3272 }
3273
3274 int auth_zones_can_fallback(struct auth_zones* az, uint8_t* nm, size_t nmlen,
3275         uint16_t dclass)
3276 {
3277         int r;
3278         struct auth_zone* z;
3279         lock_rw_rdlock(&az->lock);
3280         z = auth_zone_find(az, nm, nmlen, dclass);
3281         if(!z) {
3282                 lock_rw_unlock(&az->lock);
3283                 /* no such auth zone, fallback */
3284                 return 1;
3285         }
3286         lock_rw_rdlock(&z->lock);
3287         lock_rw_unlock(&az->lock);
3288         r = z->fallback_enabled || (!z->for_upstream);
3289         lock_rw_unlock(&z->lock);
3290         return r;
3291 }
3292
3293 int
3294 auth_zone_parse_notify_serial(sldns_buffer* pkt, uint32_t *serial)
3295 {
3296         struct query_info q;
3297         uint16_t rdlen;
3298         memset(&q, 0, sizeof(q));
3299         sldns_buffer_set_position(pkt, 0);
3300         if(!query_info_parse(&q, pkt)) return 0;
3301         if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0) return 0;
3302         /* skip name of RR in answer section */
3303         if(sldns_buffer_remaining(pkt) < 1) return 0;
3304         if(pkt_dname_len(pkt) == 0) return 0;
3305         /* check type */
3306         if(sldns_buffer_remaining(pkt) < 10 /* type,class,ttl,rdatalen*/)
3307                 return 0;
3308         if(sldns_buffer_read_u16(pkt) != LDNS_RR_TYPE_SOA) return 0;
3309         sldns_buffer_skip(pkt, 2); /* class */
3310         sldns_buffer_skip(pkt, 4); /* ttl */
3311         rdlen = sldns_buffer_read_u16(pkt); /* rdatalen */
3312         if(sldns_buffer_remaining(pkt) < rdlen) return 0;
3313         if(rdlen < 22) return 0; /* bad soa length */
3314         sldns_buffer_skip(pkt, (ssize_t)(rdlen-20));
3315         *serial = sldns_buffer_read_u32(pkt);
3316         /* return true when has serial in answer section */
3317         return 1;
3318 }
3319
3320 /** see if addr appears in the list */
3321 static int
3322 addr_in_list(struct auth_addr* list, struct sockaddr_storage* addr,
3323         socklen_t addrlen)
3324 {
3325         struct auth_addr* p;
3326         for(p=list; p; p=p->next) {
3327                 if(sockaddr_cmp_addr(addr, addrlen, &p->addr, p->addrlen)==0)
3328                         return 1;
3329         }
3330         return 0;
3331 }
3332
3333 /** check if an address matches a master specification (or one of its
3334  * addresses in the addr list) */
3335 static int
3336 addr_matches_master(struct auth_master* master, struct sockaddr_storage* addr,
3337         socklen_t addrlen, struct auth_master** fromhost)
3338 {
3339         struct sockaddr_storage a;
3340         socklen_t alen = 0;
3341         int net = 0;
3342         if(addr_in_list(master->list, addr, addrlen)) {
3343                 *fromhost = master;
3344                 return 1;
3345         }
3346         /* compare address (but not port number, that is the destination
3347          * port of the master, the port number of the received notify is
3348          * allowed to by any port on that master) */
3349         if(extstrtoaddr(master->host, &a, &alen) &&
3350                 sockaddr_cmp_addr(addr, addrlen, &a, alen)==0) {
3351                 *fromhost = master;
3352                 return 1;
3353         }
3354         /* prefixes, addr/len, like 10.0.0.0/8 */
3355         /* not http and has a / and there is one / */
3356         if(master->allow_notify && !master->http &&
3357                 strchr(master->host, '/') != NULL &&
3358                 strchr(master->host, '/') == strrchr(master->host, '/') &&
3359                 netblockstrtoaddr(master->host, UNBOUND_DNS_PORT, &a, &alen,
3360                 &net) && alen == addrlen) {
3361                 if(addr_in_common(addr, (addr_is_ip6(addr, addrlen)?128:32),
3362                         &a, net, alen) >= net) {
3363                         *fromhost = NULL; /* prefix does not have destination
3364                                 to send the probe or transfer with */
3365                         return 1; /* matches the netblock */
3366                 }
3367         }
3368         return 0;
3369 }
3370
3371 /** check access list for notifies */
3372 static int
3373 az_xfr_allowed_notify(struct auth_xfer* xfr, struct sockaddr_storage* addr,
3374         socklen_t addrlen, struct auth_master** fromhost)
3375 {
3376         struct auth_master* p;
3377         for(p=xfr->allow_notify_list; p; p=p->next) {
3378                 if(addr_matches_master(p, addr, addrlen, fromhost)) {
3379                         return 1;
3380                 }
3381         }
3382         return 0;
3383 }
3384
3385 /** see if the serial means the zone has to be updated, i.e. the serial
3386  * is newer than the zone serial, or we have no zone */
3387 static int
3388 xfr_serial_means_update(struct auth_xfer* xfr, uint32_t serial)
3389 {
3390         if(!xfr->have_zone)
3391                 return 1; /* no zone, anything is better */
3392         if(xfr->zone_expired)
3393                 return 1; /* expired, the sent serial is better than expired
3394                         data */
3395         if(compare_serial(xfr->serial, serial) < 0)
3396                 return 1; /* our serial is smaller than the sent serial,
3397                         the data is newer, fetch it */
3398         return 0;
3399 }
3400
3401 /** note notify serial, updates the notify information in the xfr struct */
3402 static void
3403 xfr_note_notify_serial(struct auth_xfer* xfr, int has_serial, uint32_t serial)
3404 {
3405         if(xfr->notify_received && xfr->notify_has_serial && has_serial) {
3406                 /* see if this serial is newer */
3407                 if(compare_serial(xfr->notify_serial, serial) < 0)
3408                         xfr->notify_serial = serial;
3409         } else if(xfr->notify_received && xfr->notify_has_serial &&
3410                 !has_serial) {
3411                 /* remove serial, we have notify without serial */
3412                 xfr->notify_has_serial = 0;
3413                 xfr->notify_serial = 0;
3414         } else if(xfr->notify_received && !xfr->notify_has_serial) {
3415                 /* we already have notify without serial, keep it
3416                  * that way; no serial check when current operation
3417                  * is done */
3418         } else {
3419                 xfr->notify_received = 1;
3420                 xfr->notify_has_serial = has_serial;
3421                 xfr->notify_serial = serial;
3422         }
3423 }
3424
3425 /** process a notify serial, start new probe or note serial. xfr is locked */
3426 static void
3427 xfr_process_notify(struct auth_xfer* xfr, struct module_env* env,
3428         int has_serial, uint32_t serial, struct auth_master* fromhost)
3429 {
3430         /* if the serial of notify is older than we have, don't fetch
3431          * a zone, we already have it */
3432         if(has_serial && !xfr_serial_means_update(xfr, serial)) {
3433                 lock_basic_unlock(&xfr->lock);
3434                 return;
3435         }
3436         /* start new probe with this addr src, or note serial */
3437         if(!xfr_start_probe(xfr, env, fromhost)) {
3438                 /* not started because already in progress, note the serial */
3439                 xfr_note_notify_serial(xfr, has_serial, serial);
3440                 lock_basic_unlock(&xfr->lock);
3441         }
3442         /* successful end of start_probe unlocked xfr->lock */
3443 }
3444
3445 int auth_zones_notify(struct auth_zones* az, struct module_env* env,
3446         uint8_t* nm, size_t nmlen, uint16_t dclass,
3447         struct sockaddr_storage* addr, socklen_t addrlen, int has_serial,
3448         uint32_t serial, int* refused)
3449 {
3450         struct auth_xfer* xfr;
3451         struct auth_master* fromhost = NULL;
3452         /* see which zone this is */
3453         lock_rw_rdlock(&az->lock);
3454         xfr = auth_xfer_find(az, nm, nmlen, dclass);
3455         if(!xfr) {
3456                 lock_rw_unlock(&az->lock);
3457                 /* no such zone, refuse the notify */
3458                 *refused = 1;
3459                 return 0;
3460         }
3461         lock_basic_lock(&xfr->lock);
3462         lock_rw_unlock(&az->lock);
3463
3464         /* check access list for notifies */
3465         if(!az_xfr_allowed_notify(xfr, addr, addrlen, &fromhost)) {
3466                 lock_basic_unlock(&xfr->lock);
3467                 /* notify not allowed, refuse the notify */
3468                 *refused = 1;
3469                 return 0;
3470         }
3471
3472         /* process the notify */
3473         xfr_process_notify(xfr, env, has_serial, serial, fromhost);
3474         return 1;
3475 }
3476
3477 int auth_zones_startprobesequence(struct auth_zones* az,
3478         struct module_env* env, uint8_t* nm, size_t nmlen, uint16_t dclass)
3479 {
3480         struct auth_xfer* xfr;
3481         lock_rw_rdlock(&az->lock);
3482         xfr = auth_xfer_find(az, nm, nmlen, dclass);
3483         if(!xfr) {
3484                 lock_rw_unlock(&az->lock);
3485                 return 0;
3486         }
3487         lock_basic_lock(&xfr->lock);
3488         lock_rw_unlock(&az->lock);
3489
3490         xfr_process_notify(xfr, env, 0, 0, NULL);
3491         return 1;
3492 }
3493
3494 /** set a zone expired */
3495 static void
3496 auth_xfer_set_expired(struct auth_xfer* xfr, struct module_env* env,
3497         int expired)
3498 {
3499         struct auth_zone* z;
3500
3501         /* expire xfr */
3502         lock_basic_lock(&xfr->lock);
3503         xfr->zone_expired = expired;
3504         lock_basic_unlock(&xfr->lock);
3505
3506         /* find auth_zone */
3507         lock_rw_rdlock(&env->auth_zones->lock);
3508         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
3509                 xfr->dclass);
3510         if(!z) {
3511                 lock_rw_unlock(&env->auth_zones->lock);
3512                 return;
3513         }
3514         lock_rw_wrlock(&z->lock);
3515         lock_rw_unlock(&env->auth_zones->lock);
3516
3517         /* expire auth_zone */
3518         z->zone_expired = expired;
3519         lock_rw_unlock(&z->lock);
3520 }
3521
3522 /** find master (from notify or probe) in list of masters */
3523 static struct auth_master*
3524 find_master_by_host(struct auth_master* list, char* host)
3525 {
3526         struct auth_master* p;
3527         for(p=list; p; p=p->next) {
3528                 if(strcmp(p->host, host) == 0)
3529                         return p;
3530         }
3531         return NULL;
3532 }
3533
3534 /** delete the looked up auth_addrs for all the masters in the list */
3535 static void
3536 xfr_masterlist_free_addrs(struct auth_master* list)
3537 {
3538         struct auth_master* m;
3539         for(m=list; m; m=m->next) {
3540                 if(m->list) {
3541                         auth_free_master_addrs(m->list);
3542                         m->list = NULL;
3543                 }
3544         }
3545 }
3546
3547 /** copy a list of auth_addrs */
3548 static struct auth_addr*
3549 auth_addr_list_copy(struct auth_addr* source)
3550 {
3551         struct auth_addr* list = NULL, *last = NULL;
3552         struct auth_addr* p;
3553         for(p=source; p; p=p->next) {
3554                 struct auth_addr* a = (struct auth_addr*)memdup(p, sizeof(*p));
3555                 if(!a) {
3556                         log_err("malloc failure");
3557                         auth_free_master_addrs(list);
3558                         return NULL;
3559                 }
3560                 a->next = NULL;
3561                 if(last) last->next = a;
3562                 if(!list) list = a;
3563                 last = a;
3564         }
3565         return list;
3566 }
3567
3568 /** copy a master to a new structure, NULL on alloc failure */
3569 static struct auth_master*
3570 auth_master_copy(struct auth_master* o)
3571 {
3572         struct auth_master* m;
3573         if(!o) return NULL;
3574         m = (struct auth_master*)memdup(o, sizeof(*o));
3575         if(!m) {
3576                 log_err("malloc failure");
3577                 return NULL;
3578         }
3579         m->next = NULL;
3580         if(m->host) {
3581                 m->host = strdup(m->host);
3582                 if(!m->host) {
3583                         free(m);
3584                         log_err("malloc failure");
3585                         return NULL;
3586                 }
3587         }
3588         if(m->file) {
3589                 m->file = strdup(m->file);
3590                 if(!m->file) {
3591                         free(m->host);
3592                         free(m);
3593                         log_err("malloc failure");
3594                         return NULL;
3595                 }
3596         }
3597         if(m->list) {
3598                 m->list = auth_addr_list_copy(m->list);
3599                 if(!m->list) {
3600                         free(m->file);
3601                         free(m->host);
3602                         free(m);
3603                         return NULL;
3604                 }
3605         }
3606         return m;
3607 }
3608
3609 /** copy the master addresses from the task_probe lookups to the allow_notify
3610  * list of masters */
3611 static void
3612 probe_copy_masters_for_allow_notify(struct auth_xfer* xfr)
3613 {
3614         struct auth_master* list = NULL, *last = NULL;
3615         struct auth_master* p;
3616         /* build up new list with copies */
3617         for(p = xfr->task_probe->masters; p; p=p->next) {
3618                 struct auth_master* m = auth_master_copy(p);
3619                 if(!m) {
3620                         auth_free_masters(list);
3621                         /* failed because of malloc failure, use old list */
3622                         return;
3623                 }
3624                 m->next = NULL;
3625                 if(last) last->next = m;
3626                 if(!list) list = m;
3627                 last = m;
3628         }
3629         /* success, replace list */
3630         auth_free_masters(xfr->allow_notify_list);
3631         xfr->allow_notify_list = list;
3632 }
3633
3634 /** start the lookups for task_transfer */
3635 static void
3636 xfr_transfer_start_lookups(struct auth_xfer* xfr)
3637 {
3638         /* delete all the looked up addresses in the list */
3639         xfr_masterlist_free_addrs(xfr->task_transfer->masters);
3640
3641         /* start lookup at the first master */
3642         xfr->task_transfer->lookup_target = xfr->task_transfer->masters;
3643         xfr->task_transfer->lookup_aaaa = 0;
3644 }
3645
3646 /** move to the next lookup of hostname for task_transfer */
3647 static void
3648 xfr_transfer_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3649 {
3650         if(!xfr->task_transfer->lookup_target)
3651                 return; /* already at end of list */
3652         if(!xfr->task_transfer->lookup_aaaa && env->cfg->do_ip6) {
3653                 /* move to lookup AAAA */
3654                 xfr->task_transfer->lookup_aaaa = 1;
3655                 return;
3656         }
3657         xfr->task_transfer->lookup_target =
3658                 xfr->task_transfer->lookup_target->next;
3659         xfr->task_transfer->lookup_aaaa = 0;
3660         if(!env->cfg->do_ip4 && xfr->task_transfer->lookup_target!=NULL)
3661                 xfr->task_transfer->lookup_aaaa = 1;
3662 }
3663
3664 /** start the lookups for task_probe */
3665 static void
3666 xfr_probe_start_lookups(struct auth_xfer* xfr)
3667 {
3668         /* delete all the looked up addresses in the list */
3669         xfr_masterlist_free_addrs(xfr->task_probe->masters);
3670
3671         /* start lookup at the first master */
3672         xfr->task_probe->lookup_target = xfr->task_probe->masters;
3673         xfr->task_probe->lookup_aaaa = 0;
3674 }
3675
3676 /** move to the next lookup of hostname for task_probe */
3677 static void
3678 xfr_probe_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3679 {
3680         if(!xfr->task_probe->lookup_target)
3681                 return; /* already at end of list */
3682         if(!xfr->task_probe->lookup_aaaa && env->cfg->do_ip6) {
3683                 /* move to lookup AAAA */
3684                 xfr->task_probe->lookup_aaaa = 1;
3685                 return;
3686         }
3687         xfr->task_probe->lookup_target = xfr->task_probe->lookup_target->next;
3688         xfr->task_probe->lookup_aaaa = 0;
3689         if(!env->cfg->do_ip4 && xfr->task_probe->lookup_target!=NULL)
3690                 xfr->task_probe->lookup_aaaa = 1;
3691 }
3692
3693 /** start the iteration of the task_transfer list of masters */
3694 static void
3695 xfr_transfer_start_list(struct auth_xfer* xfr, struct auth_master* spec)
3696 {
3697         if(spec) {
3698                 xfr->task_transfer->scan_specific = find_master_by_host(
3699                         xfr->task_transfer->masters, spec->host);
3700                 if(xfr->task_transfer->scan_specific) {
3701                         xfr->task_transfer->scan_target = NULL;
3702                         xfr->task_transfer->scan_addr = NULL;
3703                         if(xfr->task_transfer->scan_specific->list)
3704                                 xfr->task_transfer->scan_addr =
3705                                         xfr->task_transfer->scan_specific->list;
3706                         return;
3707                 }
3708         }
3709         /* no specific (notified) host to scan */
3710         xfr->task_transfer->scan_specific = NULL;
3711         xfr->task_transfer->scan_addr = NULL;
3712         /* pick up first scan target */
3713         xfr->task_transfer->scan_target = xfr->task_transfer->masters;
3714         if(xfr->task_transfer->scan_target && xfr->task_transfer->
3715                 scan_target->list)
3716                 xfr->task_transfer->scan_addr =
3717                         xfr->task_transfer->scan_target->list;
3718 }
3719
3720 /** start the iteration of the task_probe list of masters */
3721 static void
3722 xfr_probe_start_list(struct auth_xfer* xfr, struct auth_master* spec)
3723 {
3724         if(spec) {
3725                 xfr->task_probe->scan_specific = find_master_by_host(
3726                         xfr->task_probe->masters, spec->host);
3727                 if(xfr->task_probe->scan_specific) {
3728                         xfr->task_probe->scan_target = NULL;
3729                         xfr->task_probe->scan_addr = NULL;
3730                         if(xfr->task_probe->scan_specific->list)
3731                                 xfr->task_probe->scan_addr =
3732                                         xfr->task_probe->scan_specific->list;
3733                         return;
3734                 }
3735         }
3736         /* no specific (notified) host to scan */
3737         xfr->task_probe->scan_specific = NULL;
3738         xfr->task_probe->scan_addr = NULL;
3739         /* pick up first scan target */
3740         xfr->task_probe->scan_target = xfr->task_probe->masters;
3741         if(xfr->task_probe->scan_target && xfr->task_probe->scan_target->list)
3742                 xfr->task_probe->scan_addr =
3743                         xfr->task_probe->scan_target->list;
3744 }
3745
3746 /** pick up the master that is being scanned right now, task_transfer */
3747 static struct auth_master*
3748 xfr_transfer_current_master(struct auth_xfer* xfr)
3749 {
3750         if(xfr->task_transfer->scan_specific)
3751                 return xfr->task_transfer->scan_specific;
3752         return xfr->task_transfer->scan_target;
3753 }
3754
3755 /** pick up the master that is being scanned right now, task_probe */
3756 static struct auth_master*
3757 xfr_probe_current_master(struct auth_xfer* xfr)
3758 {
3759         if(xfr->task_probe->scan_specific)
3760                 return xfr->task_probe->scan_specific;
3761         return xfr->task_probe->scan_target;
3762 }
3763
3764 /** true if at end of list, task_transfer */
3765 static int
3766 xfr_transfer_end_of_list(struct auth_xfer* xfr)
3767 {
3768         return !xfr->task_transfer->scan_specific &&
3769                 !xfr->task_transfer->scan_target;
3770 }
3771
3772 /** true if at end of list, task_probe */
3773 static int
3774 xfr_probe_end_of_list(struct auth_xfer* xfr)
3775 {
3776         return !xfr->task_probe->scan_specific && !xfr->task_probe->scan_target;
3777 }
3778
3779 /** move to next master in list, task_transfer */
3780 static void
3781 xfr_transfer_nextmaster(struct auth_xfer* xfr)
3782 {
3783         if(!xfr->task_transfer->scan_specific &&
3784                 !xfr->task_transfer->scan_target)
3785                 return;
3786         if(xfr->task_transfer->scan_addr) {
3787                 xfr->task_transfer->scan_addr =
3788                         xfr->task_transfer->scan_addr->next;
3789                 if(xfr->task_transfer->scan_addr)
3790                         return;
3791         }
3792         if(xfr->task_transfer->scan_specific) {
3793                 xfr->task_transfer->scan_specific = NULL;
3794                 xfr->task_transfer->scan_target = xfr->task_transfer->masters;
3795                 if(xfr->task_transfer->scan_target && xfr->task_transfer->
3796                         scan_target->list)
3797                         xfr->task_transfer->scan_addr =
3798                                 xfr->task_transfer->scan_target->list;
3799                 return;
3800         }
3801         if(!xfr->task_transfer->scan_target)
3802                 return;
3803         xfr->task_transfer->scan_target = xfr->task_transfer->scan_target->next;
3804         if(xfr->task_transfer->scan_target && xfr->task_transfer->
3805                 scan_target->list)
3806                 xfr->task_transfer->scan_addr =
3807                         xfr->task_transfer->scan_target->list;
3808         return;
3809 }
3810
3811 /** move to next master in list, task_probe */
3812 static void
3813 xfr_probe_nextmaster(struct auth_xfer* xfr)
3814 {
3815         if(!xfr->task_probe->scan_specific && !xfr->task_probe->scan_target)
3816                 return;
3817         if(xfr->task_probe->scan_addr) {
3818                 xfr->task_probe->scan_addr = xfr->task_probe->scan_addr->next;
3819                 if(xfr->task_probe->scan_addr)
3820                         return;
3821         }
3822         if(xfr->task_probe->scan_specific) {
3823                 xfr->task_probe->scan_specific = NULL;
3824                 xfr->task_probe->scan_target = xfr->task_probe->masters;
3825                 if(xfr->task_probe->scan_target && xfr->task_probe->
3826                         scan_target->list)
3827                         xfr->task_probe->scan_addr =
3828                                 xfr->task_probe->scan_target->list;
3829                 return;
3830         }
3831         if(!xfr->task_probe->scan_target)
3832                 return;
3833         xfr->task_probe->scan_target = xfr->task_probe->scan_target->next;
3834         if(xfr->task_probe->scan_target && xfr->task_probe->
3835                 scan_target->list)
3836                 xfr->task_probe->scan_addr =
3837                         xfr->task_probe->scan_target->list;
3838         return;
3839 }
3840
3841 /** create SOA probe packet for xfr */
3842 static void
3843 xfr_create_soa_probe_packet(struct auth_xfer* xfr, sldns_buffer* buf,
3844         uint16_t id)
3845 {
3846         struct query_info qinfo;
3847
3848         memset(&qinfo, 0, sizeof(qinfo));
3849         qinfo.qname = xfr->name;
3850         qinfo.qname_len = xfr->namelen;
3851         qinfo.qtype = LDNS_RR_TYPE_SOA;
3852         qinfo.qclass = xfr->dclass;
3853         qinfo_query_encode(buf, &qinfo);
3854         sldns_buffer_write_u16_at(buf, 0, id);
3855 }
3856
3857 /** create IXFR/AXFR packet for xfr */
3858 static void
3859 xfr_create_ixfr_packet(struct auth_xfer* xfr, sldns_buffer* buf, uint16_t id,
3860         struct auth_master* master)
3861 {
3862         struct query_info qinfo;
3863         uint32_t serial;
3864         int have_zone;
3865         have_zone = xfr->have_zone;
3866         serial = xfr->serial;
3867
3868         memset(&qinfo, 0, sizeof(qinfo));
3869         qinfo.qname = xfr->name;
3870         qinfo.qname_len = xfr->namelen;
3871         xfr->task_transfer->got_xfr_serial = 0;
3872         xfr->task_transfer->rr_scan_num = 0;
3873         xfr->task_transfer->incoming_xfr_serial = 0;
3874         xfr->task_transfer->on_ixfr_is_axfr = 0;
3875         xfr->task_transfer->on_ixfr = 1;
3876         qinfo.qtype = LDNS_RR_TYPE_IXFR;
3877         if(!have_zone || xfr->task_transfer->ixfr_fail || !master->ixfr) {
3878                 qinfo.qtype = LDNS_RR_TYPE_AXFR;
3879                 xfr->task_transfer->ixfr_fail = 0;
3880                 xfr->task_transfer->on_ixfr = 0;
3881         }
3882
3883         qinfo.qclass = xfr->dclass;
3884         qinfo_query_encode(buf, &qinfo);
3885         sldns_buffer_write_u16_at(buf, 0, id);
3886
3887         /* append serial for IXFR */
3888         if(qinfo.qtype == LDNS_RR_TYPE_IXFR) {
3889                 size_t end = sldns_buffer_limit(buf);
3890                 sldns_buffer_clear(buf);
3891                 sldns_buffer_set_position(buf, end);
3892                 /* auth section count 1 */
3893                 sldns_buffer_write_u16_at(buf, LDNS_NSCOUNT_OFF, 1);
3894                 /* write SOA */
3895                 sldns_buffer_write_u8(buf, 0xC0); /* compressed ptr to qname */
3896                 sldns_buffer_write_u8(buf, 0x0C);
3897                 sldns_buffer_write_u16(buf, LDNS_RR_TYPE_SOA);
3898                 sldns_buffer_write_u16(buf, qinfo.qclass);
3899                 sldns_buffer_write_u32(buf, 0); /* ttl */
3900                 sldns_buffer_write_u16(buf, 22); /* rdata length */
3901                 sldns_buffer_write_u8(buf, 0); /* . */
3902                 sldns_buffer_write_u8(buf, 0); /* . */
3903                 sldns_buffer_write_u32(buf, serial); /* serial */
3904                 sldns_buffer_write_u32(buf, 0); /* refresh */
3905                 sldns_buffer_write_u32(buf, 0); /* retry */
3906                 sldns_buffer_write_u32(buf, 0); /* expire */
3907                 sldns_buffer_write_u32(buf, 0); /* minimum */
3908                 sldns_buffer_flip(buf);
3909         }
3910 }
3911
3912 /** check if returned packet is OK */
3913 static int
3914 check_packet_ok(sldns_buffer* pkt, uint16_t qtype, struct auth_xfer* xfr,
3915         uint32_t* serial)
3916 {
3917         /* parse to see if packet worked, valid reply */
3918
3919         /* check serial number of SOA */
3920         if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE)
3921                 return 0;
3922
3923         /* check ID */
3924         if(LDNS_ID_WIRE(sldns_buffer_begin(pkt)) != xfr->task_probe->id)
3925                 return 0;
3926
3927         /* check flag bits and rcode */
3928         if(!LDNS_QR_WIRE(sldns_buffer_begin(pkt)))
3929                 return 0;
3930         if(LDNS_OPCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_PACKET_QUERY)
3931                 return 0;
3932         if(LDNS_RCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_RCODE_NOERROR)
3933                 return 0;
3934
3935         /* check qname */
3936         if(LDNS_QDCOUNT(sldns_buffer_begin(pkt)) != 1)
3937                 return 0;
3938         sldns_buffer_skip(pkt, LDNS_HEADER_SIZE);
3939         if(sldns_buffer_remaining(pkt) < xfr->namelen)
3940                 return 0;
3941         if(query_dname_compare(sldns_buffer_current(pkt), xfr->name) != 0)
3942                 return 0;
3943         sldns_buffer_skip(pkt, (ssize_t)xfr->namelen);
3944
3945         /* check qtype, qclass */
3946         if(sldns_buffer_remaining(pkt) < 4)
3947                 return 0;
3948         if(sldns_buffer_read_u16(pkt) != qtype)
3949                 return 0;
3950         if(sldns_buffer_read_u16(pkt) != xfr->dclass)
3951                 return 0;
3952
3953         if(serial) {
3954                 uint16_t rdlen;
3955                 /* read serial number, from answer section SOA */
3956                 if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0)
3957                         return 0;
3958                 /* read from first record SOA record */
3959                 if(sldns_buffer_remaining(pkt) < 1)
3960                         return 0;
3961                 if(dname_pkt_compare(pkt, sldns_buffer_current(pkt),
3962                         xfr->name) != 0)
3963                         return 0;
3964                 if(!pkt_dname_len(pkt))
3965                         return 0;
3966                 /* type, class, ttl, rdatalen */
3967                 if(sldns_buffer_remaining(pkt) < 4+4+2)
3968                         return 0;
3969                 if(sldns_buffer_read_u16(pkt) != qtype)
3970                         return 0;
3971                 if(sldns_buffer_read_u16(pkt) != xfr->dclass)
3972                         return 0;
3973                 sldns_buffer_skip(pkt, 4); /* ttl */
3974                 rdlen = sldns_buffer_read_u16(pkt);
3975                 if(sldns_buffer_remaining(pkt) < rdlen)
3976                         return 0;
3977                 if(sldns_buffer_remaining(pkt) < 1)
3978                         return 0;
3979                 if(!pkt_dname_len(pkt)) /* soa name */
3980                         return 0;
3981                 if(sldns_buffer_remaining(pkt) < 1)
3982                         return 0;
3983                 if(!pkt_dname_len(pkt)) /* soa name */
3984                         return 0;
3985                 if(sldns_buffer_remaining(pkt) < 20)
3986                         return 0;
3987                 *serial = sldns_buffer_read_u32(pkt);
3988         }
3989         return 1;
3990 }
3991
3992 /** read one line from chunks into buffer at current position */
3993 static int
3994 chunkline_get_line(struct auth_chunk** chunk, size_t* chunk_pos,
3995         sldns_buffer* buf)
3996 {
3997         int readsome = 0;
3998         while(*chunk) {
3999                 /* more text in this chunk? */
4000                 if(*chunk_pos < (*chunk)->len) {
4001                         readsome = 1;
4002                         while(*chunk_pos < (*chunk)->len) {
4003                                 char c = (char)((*chunk)->data[*chunk_pos]);
4004                                 (*chunk_pos)++;
4005                                 if(sldns_buffer_remaining(buf) < 2) {
4006                                         /* buffer too short */
4007                                         verbose(VERB_ALGO, "http chunkline, "
4008                                                 "line too long");
4009                                         return 0;
4010                                 }
4011                                 sldns_buffer_write_u8(buf, (uint8_t)c);
4012                                 if(c == '\n') {
4013                                         /* we are done */
4014                                         return 1;
4015                                 }
4016                         }
4017                 }
4018                 /* move to next chunk */
4019                 *chunk = (*chunk)->next;
4020                 *chunk_pos = 0;
4021         }
4022         /* no more text */
4023         if(readsome) return 1;
4024         return 0;
4025 }
4026
4027 /** count number of open and closed parenthesis in a chunkline */
4028 static int
4029 chunkline_count_parens(sldns_buffer* buf, size_t start)
4030 {
4031         size_t end = sldns_buffer_position(buf);
4032         size_t i;
4033         int count = 0;
4034         int squote = 0, dquote = 0;
4035         for(i=start; i<end; i++) {
4036                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4037                 if(squote && c != '\'') continue;
4038                 if(dquote && c != '"') continue;
4039                 if(c == '"')
4040                         dquote = !dquote; /* skip quoted part */
4041                 else if(c == '\'')
4042                         squote = !squote; /* skip quoted part */
4043                 else if(c == '(')
4044                         count ++;
4045                 else if(c == ')')
4046                         count --;
4047                 else if(c == ';') {
4048                         /* rest is a comment */
4049                         return count;
4050                 }
4051         }
4052         return count;
4053 }
4054
4055 /** remove trailing ;... comment from a line in the chunkline buffer */
4056 static void
4057 chunkline_remove_trailcomment(sldns_buffer* buf, size_t start)
4058 {
4059         size_t end = sldns_buffer_position(buf);
4060         size_t i;
4061         int squote = 0, dquote = 0;
4062         for(i=start; i<end; i++) {
4063                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4064                 if(squote && c != '\'') continue;
4065                 if(dquote && c != '"') continue;
4066                 if(c == '"')
4067                         dquote = !dquote; /* skip quoted part */
4068                 else if(c == '\'')
4069                         squote = !squote; /* skip quoted part */
4070                 else if(c == ';') {
4071                         /* rest is a comment */
4072                         sldns_buffer_set_position(buf, i);
4073                         return;
4074                 }
4075         }
4076         /* nothing to remove */
4077 }
4078
4079 /** see if a chunkline is a comment line (or empty line) */
4080 static int
4081 chunkline_is_comment_line_or_empty(sldns_buffer* buf)
4082 {
4083         size_t i, end = sldns_buffer_limit(buf);
4084         for(i=0; i<end; i++) {
4085                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4086                 if(c == ';')
4087                         return 1; /* comment */
4088                 else if(c != ' ' && c != '\t' && c != '\r' && c != '\n')
4089                         return 0; /* not a comment */
4090         }
4091         return 1; /* empty */
4092 }
4093
4094 /** find a line with ( ) collated */
4095 static int
4096 chunkline_get_line_collated(struct auth_chunk** chunk, size_t* chunk_pos,
4097         sldns_buffer* buf)
4098 {
4099         size_t pos;
4100         int parens = 0;
4101         sldns_buffer_clear(buf);
4102         pos = sldns_buffer_position(buf);
4103         if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4104                 if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4105                         sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4106                 else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4107                 sldns_buffer_flip(buf);
4108                 return 0;
4109         }
4110         parens += chunkline_count_parens(buf, pos);
4111         while(parens > 0) {
4112                 chunkline_remove_trailcomment(buf, pos);
4113                 pos = sldns_buffer_position(buf);
4114                 if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4115                         if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4116                                 sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4117                         else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4118                         sldns_buffer_flip(buf);
4119                         return 0;
4120                 }
4121                 parens += chunkline_count_parens(buf, pos);
4122         }
4123
4124         if(sldns_buffer_remaining(buf) < 1) {
4125                 verbose(VERB_ALGO, "http chunkline: "
4126                         "line too long");
4127                 return 0;
4128         }
4129         sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4130         sldns_buffer_flip(buf);
4131         return 1;
4132 }
4133
4134 /** process $ORIGIN for http */
4135 static int
4136 http_parse_origin(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4137 {
4138         char* line = (char*)sldns_buffer_begin(buf);
4139         if(strncmp(line, "$ORIGIN", 7) == 0 &&
4140                 isspace((unsigned char)line[7])) {
4141                 int s;
4142                 pstate->origin_len = sizeof(pstate->origin);
4143                 s = sldns_str2wire_dname_buf(sldns_strip_ws(line+8),
4144                         pstate->origin, &pstate->origin_len);
4145                 if(s) pstate->origin_len = 0;
4146                 return 1;
4147         }
4148         return 0;
4149 }
4150
4151 /** process $TTL for http */
4152 static int
4153 http_parse_ttl(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4154 {
4155         char* line = (char*)sldns_buffer_begin(buf);
4156         if(strncmp(line, "$TTL", 4) == 0 &&
4157                 isspace((unsigned char)line[4])) {
4158                 const char* end = NULL;
4159                 pstate->default_ttl = sldns_str2period(
4160                         sldns_strip_ws(line+5), &end);
4161                 return 1;
4162         }
4163         return 0;
4164 }
4165
4166 /** find noncomment RR line in chunks, collates lines if ( ) format */
4167 static int
4168 chunkline_non_comment_RR(struct auth_chunk** chunk, size_t* chunk_pos,
4169         sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4170 {
4171         while(chunkline_get_line_collated(chunk, chunk_pos, buf)) {
4172                 if(chunkline_is_comment_line_or_empty(buf)) {
4173                         /* a comment, go to next line */
4174                         continue;
4175                 }
4176                 if(http_parse_origin(buf, pstate)) {
4177                         continue; /* $ORIGIN has been handled */
4178                 }
4179                 if(http_parse_ttl(buf, pstate)) {
4180                         continue; /* $TTL has been handled */
4181                 }
4182                 return 1;
4183         }
4184         /* no noncomments, fail */
4185         return 0;
4186 }
4187
4188 /** check syntax of chunklist zonefile, parse first RR, return false on
4189  * failure and return a string in the scratch buffer (first RR string)
4190  * on failure. */
4191 static int
4192 http_zonefile_syntax_check(struct auth_xfer* xfr, sldns_buffer* buf)
4193 {
4194         uint8_t rr[LDNS_RR_BUF_SIZE];
4195         size_t rr_len, dname_len = 0;
4196         struct sldns_file_parse_state pstate;
4197         struct auth_chunk* chunk;
4198         size_t chunk_pos;
4199         int e;
4200         memset(&pstate, 0, sizeof(pstate));
4201         pstate.default_ttl = 3600;
4202         if(xfr->namelen < sizeof(pstate.origin)) {
4203                 pstate.origin_len = xfr->namelen;
4204                 memmove(pstate.origin, xfr->name, xfr->namelen);
4205         }
4206         chunk = xfr->task_transfer->chunks_first;
4207         chunk_pos = 0;
4208         if(!chunkline_non_comment_RR(&chunk, &chunk_pos, buf, &pstate)) {
4209                 return 0;
4210         }
4211         rr_len = sizeof(rr);
4212         e=sldns_str2wire_rr_buf((char*)sldns_buffer_begin(buf), rr, &rr_len,
4213                 &dname_len, pstate.default_ttl,
4214                 pstate.origin_len?pstate.origin:NULL, pstate.origin_len,
4215                 pstate.prev_rr_len?pstate.prev_rr:NULL, pstate.prev_rr_len);
4216         if(e != 0) {
4217                 log_err("parse failure on first RR[%d]: %s",
4218                         LDNS_WIREPARSE_OFFSET(e),
4219                         sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)));
4220                 return 0;
4221         }
4222         /* check that class is correct */
4223         if(sldns_wirerr_get_class(rr, rr_len, dname_len) != xfr->dclass) {
4224                 log_err("parse failure: first record in downloaded zonefile "
4225                         "from wrong RR class");
4226                 return 0;
4227         }
4228         return 1;
4229 }
4230
4231 /** sum sizes of chunklist */
4232 static size_t
4233 chunklist_sum(struct auth_chunk* list)
4234 {
4235         struct auth_chunk* p;
4236         size_t s = 0;
4237         for(p=list; p; p=p->next) {
4238                 s += p->len;
4239         }
4240         return s;
4241 }
4242
4243 /** remove newlines from collated line */
4244 static void
4245 chunkline_newline_removal(sldns_buffer* buf)
4246 {
4247         size_t i, end=sldns_buffer_limit(buf);
4248         for(i=0; i<end; i++) {
4249                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4250                 if(c == '\n' && i==end-1) {
4251                         sldns_buffer_write_u8_at(buf, i, 0);
4252                         sldns_buffer_set_limit(buf, end-1);
4253                         return;
4254                 }
4255                 if(c == '\n')
4256                         sldns_buffer_write_u8_at(buf, i, (uint8_t)' ');
4257         }
4258 }
4259
4260 /** for http download, parse and add RR to zone */
4261 static int
4262 http_parse_add_rr(struct auth_xfer* xfr, struct auth_zone* z,
4263         sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4264 {
4265         uint8_t rr[LDNS_RR_BUF_SIZE];
4266         size_t rr_len, dname_len = 0;
4267         int e;
4268         char* line = (char*)sldns_buffer_begin(buf);
4269         rr_len = sizeof(rr);
4270         e = sldns_str2wire_rr_buf(line, rr, &rr_len, &dname_len,
4271                 pstate->default_ttl,
4272                 pstate->origin_len?pstate->origin:NULL, pstate->origin_len,
4273                 pstate->prev_rr_len?pstate->prev_rr:NULL, pstate->prev_rr_len);
4274         if(e != 0) {
4275                 log_err("%s/%s parse failure RR[%d]: %s in '%s'",
4276                         xfr->task_transfer->master->host,
4277                         xfr->task_transfer->master->file,
4278                         LDNS_WIREPARSE_OFFSET(e),
4279                         sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)),
4280                         line);
4281                 return 0;
4282         }
4283         if(rr_len == 0)
4284                 return 1; /* empty line or so */
4285
4286         /* set prev */
4287         if(dname_len < sizeof(pstate->prev_rr)) {
4288                 memmove(pstate->prev_rr, rr, dname_len);
4289                 pstate->prev_rr_len = dname_len;
4290         }
4291
4292         return az_insert_rr(z, rr, rr_len, dname_len, NULL);
4293 }
4294
4295 /** RR list iterator, returns RRs from answer section one by one from the
4296  * dns packets in the chunklist */
4297 static void
4298 chunk_rrlist_start(struct auth_xfer* xfr, struct auth_chunk** rr_chunk,
4299         int* rr_num, size_t* rr_pos)
4300 {
4301         *rr_chunk = xfr->task_transfer->chunks_first;
4302         *rr_num = 0;
4303         *rr_pos = 0;
4304 }
4305
4306 /** RR list iterator, see if we are at the end of the list */
4307 static int
4308 chunk_rrlist_end(struct auth_chunk* rr_chunk, int rr_num)
4309 {
4310         while(rr_chunk) {
4311                 if(rr_chunk->len < LDNS_HEADER_SIZE)
4312                         return 1;
4313                 if(rr_num < (int)LDNS_ANCOUNT(rr_chunk->data))
4314                         return 0;
4315                 /* no more RRs in this chunk */
4316                 /* continue with next chunk, see if it has RRs */
4317                 rr_chunk = rr_chunk->next;
4318                 rr_num = 0;
4319         }
4320         return 1;
4321 }
4322
4323 /** RR list iterator, move to next RR */
4324 static void
4325 chunk_rrlist_gonext(struct auth_chunk** rr_chunk, int* rr_num,
4326         size_t* rr_pos, size_t rr_nextpos)
4327 {
4328         /* already at end of chunks? */
4329         if(!*rr_chunk)
4330                 return;
4331         /* move within this chunk */
4332         if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4333                 (*rr_num)+1 < (int)LDNS_ANCOUNT((*rr_chunk)->data)) {
4334                 (*rr_num) += 1;
4335                 *rr_pos = rr_nextpos;
4336                 return;
4337         }
4338         /* no more RRs in this chunk */
4339         /* continue with next chunk, see if it has RRs */
4340         if(*rr_chunk)
4341                 *rr_chunk = (*rr_chunk)->next;
4342         while(*rr_chunk) {
4343                 *rr_num = 0;
4344                 *rr_pos = 0;
4345                 if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4346                         LDNS_ANCOUNT((*rr_chunk)->data) > 0) {
4347                         return;
4348                 }
4349                 *rr_chunk = (*rr_chunk)->next;
4350         }
4351 }
4352
4353 /** RR iterator, get current RR information, false on parse error */
4354 static int
4355 chunk_rrlist_get_current(struct auth_chunk* rr_chunk, int rr_num,
4356         size_t rr_pos, uint8_t** rr_dname, uint16_t* rr_type,
4357         uint16_t* rr_class, uint32_t* rr_ttl, uint16_t* rr_rdlen,
4358         uint8_t** rr_rdata, size_t* rr_nextpos)
4359 {
4360         sldns_buffer pkt;
4361         /* integrity checks on position */
4362         if(!rr_chunk) return 0;
4363         if(rr_chunk->len < LDNS_HEADER_SIZE) return 0;
4364         if(rr_num >= (int)LDNS_ANCOUNT(rr_chunk->data)) return 0;
4365         if(rr_pos >= rr_chunk->len) return 0;
4366
4367         /* fetch rr information */
4368         sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4369         if(rr_pos == 0) {
4370                 size_t i;
4371                 /* skip question section */
4372                 sldns_buffer_set_position(&pkt, LDNS_HEADER_SIZE);
4373                 for(i=0; i<LDNS_QDCOUNT(rr_chunk->data); i++) {
4374                         if(pkt_dname_len(&pkt) == 0) return 0;
4375                         if(sldns_buffer_remaining(&pkt) < 4) return 0;
4376                         sldns_buffer_skip(&pkt, 4); /* type and class */
4377                 }
4378         } else  {
4379                 sldns_buffer_set_position(&pkt, rr_pos);
4380         }
4381         *rr_dname = sldns_buffer_current(&pkt);
4382         if(pkt_dname_len(&pkt) == 0) return 0;
4383         if(sldns_buffer_remaining(&pkt) < 10) return 0;
4384         *rr_type = sldns_buffer_read_u16(&pkt);
4385         *rr_class = sldns_buffer_read_u16(&pkt);
4386         *rr_ttl = sldns_buffer_read_u32(&pkt);
4387         *rr_rdlen = sldns_buffer_read_u16(&pkt);
4388         if(sldns_buffer_remaining(&pkt) < (*rr_rdlen)) return 0;
4389         *rr_rdata = sldns_buffer_current(&pkt);
4390         sldns_buffer_skip(&pkt, (ssize_t)(*rr_rdlen));
4391         *rr_nextpos = sldns_buffer_position(&pkt);
4392         return 1;
4393 }
4394
4395 /** print log message where we are in parsing the zone transfer */
4396 static void
4397 log_rrlist_position(const char* label, struct auth_chunk* rr_chunk,
4398         uint8_t* rr_dname, uint16_t rr_type, size_t rr_counter)
4399 {
4400         sldns_buffer pkt;
4401         size_t dlen;
4402         uint8_t buf[256];
4403         char str[256];
4404         char typestr[32];
4405         sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4406         sldns_buffer_set_position(&pkt, (size_t)(rr_dname -
4407                 sldns_buffer_begin(&pkt)));
4408         if((dlen=pkt_dname_len(&pkt)) == 0) return;
4409         if(dlen >= sizeof(buf)) return;
4410         dname_pkt_copy(&pkt, buf, rr_dname);
4411         dname_str(buf, str);
4412         (void)sldns_wire2str_type_buf(rr_type, typestr, sizeof(typestr));
4413         verbose(VERB_ALGO, "%s at[%d] %s %s", label, (int)rr_counter,
4414                 str, typestr);
4415 }
4416
4417 /** check that start serial is OK for ixfr. we are at rr_counter == 0,
4418  * and we are going to check rr_counter == 1 (has to be type SOA) serial */
4419 static int
4420 ixfr_start_serial(struct auth_chunk* rr_chunk, int rr_num, size_t rr_pos,
4421         uint8_t* rr_dname, uint16_t rr_type, uint16_t rr_class,
4422         uint32_t rr_ttl, uint16_t rr_rdlen, uint8_t* rr_rdata,
4423         size_t rr_nextpos, uint32_t transfer_serial, uint32_t xfr_serial)
4424 {
4425         uint32_t startserial;
4426         /* move forward on RR */
4427         chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4428         if(chunk_rrlist_end(rr_chunk, rr_num)) {
4429                 /* no second SOA */
4430                 verbose(VERB_OPS, "IXFR has no second SOA record");
4431                 return 0;
4432         }
4433         if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4434                 &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4435                 &rr_rdata, &rr_nextpos)) {
4436                 verbose(VERB_OPS, "IXFR cannot parse second SOA record");
4437                 /* failed to parse RR */
4438                 return 0;
4439         }
4440         if(rr_type != LDNS_RR_TYPE_SOA) {
4441                 verbose(VERB_OPS, "IXFR second record is not type SOA");
4442                 return 0;
4443         }
4444         if(rr_rdlen < 22) {
4445                 verbose(VERB_OPS, "IXFR, second SOA has short rdlength");
4446                 return 0; /* bad SOA rdlen */
4447         }
4448         startserial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4449         if(startserial == transfer_serial) {
4450                 /* empty AXFR, not an IXFR */
4451                 verbose(VERB_OPS, "IXFR second serial same as first");
4452                 return 0;
4453         }
4454         if(startserial != xfr_serial) {
4455                 /* wrong start serial, it does not match the serial in
4456                  * memory */
4457                 verbose(VERB_OPS, "IXFR is from serial %u to %u but %u "
4458                         "in memory, rejecting the zone transfer",
4459                         (unsigned)startserial, (unsigned)transfer_serial,
4460                         (unsigned)xfr_serial);
4461                 return 0;
4462         }
4463         /* everything OK in second SOA serial */
4464         return 1;
4465 }
4466
4467 /** apply IXFR to zone in memory. z is locked. false on failure(mallocfail) */
4468 static int
4469 apply_ixfr(struct auth_xfer* xfr, struct auth_zone* z,
4470         struct sldns_buffer* scratch_buffer)
4471 {
4472         struct auth_chunk* rr_chunk;
4473         int rr_num;
4474         size_t rr_pos;
4475         uint8_t* rr_dname, *rr_rdata;
4476         uint16_t rr_type, rr_class, rr_rdlen;
4477         uint32_t rr_ttl;
4478         size_t rr_nextpos;
4479         int have_transfer_serial = 0;
4480         uint32_t transfer_serial = 0;
4481         size_t rr_counter = 0;
4482         int delmode = 0;
4483         int softfail = 0;
4484
4485         /* start RR iterator over chunklist of packets */
4486         chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4487         while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4488                 if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4489                         &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4490                         &rr_rdata, &rr_nextpos)) {
4491                         /* failed to parse RR */
4492                         return 0;
4493                 }
4494                 if(verbosity>=7) log_rrlist_position("apply ixfr",
4495                         rr_chunk, rr_dname, rr_type, rr_counter);
4496                 /* twiddle add/del mode and check for start and end */
4497                 if(rr_counter == 0 && rr_type != LDNS_RR_TYPE_SOA)
4498                         return 0;
4499                 if(rr_counter == 1 && rr_type != LDNS_RR_TYPE_SOA) {
4500                         /* this is an AXFR returned from the IXFR master */
4501                         /* but that should already have been detected, by
4502                          * on_ixfr_is_axfr */
4503                         return 0;
4504                 }
4505                 if(rr_type == LDNS_RR_TYPE_SOA) {
4506                         uint32_t serial;
4507                         if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4508                         serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4509                         if(have_transfer_serial == 0) {
4510                                 have_transfer_serial = 1;
4511                                 transfer_serial = serial;
4512                                 delmode = 1; /* gets negated below */
4513                                 /* check second RR before going any further */
4514                                 if(!ixfr_start_serial(rr_chunk, rr_num, rr_pos,
4515                                         rr_dname, rr_type, rr_class, rr_ttl,
4516                                         rr_rdlen, rr_rdata, rr_nextpos,
4517                                         transfer_serial, xfr->serial)) {
4518                                         return 0;
4519                                 }
4520                         } else if(transfer_serial == serial) {
4521                                 have_transfer_serial++;
4522                                 if(rr_counter == 1) {
4523                                         /* empty AXFR, with SOA; SOA; */
4524                                         /* should have been detected by
4525                                          * on_ixfr_is_axfr */
4526                                         return 0;
4527                                 }
4528                                 if(have_transfer_serial == 3) {
4529                                         /* see serial three times for end */
4530                                         /* eg. IXFR:
4531                                          *  SOA 3 start
4532                                          *  SOA 1 second RR, followed by del
4533                                          *  SOA 2 followed by add
4534                                          *  SOA 2 followed by del
4535                                          *  SOA 3 followed by add
4536                                          *  SOA 3 end */
4537                                         /* ended by SOA record */
4538                                         xfr->serial = transfer_serial;
4539                                         break;
4540                                 }
4541                         }
4542                         /* twiddle add/del mode */
4543                         /* switch from delete part to add part and back again
4544                          * just before the soa, it gets deleted and added too
4545                          * this means we switch to delete mode for the final
4546                          * SOA(so skip that one) */
4547                         delmode = !delmode;
4548                 }
4549                 /* process this RR */
4550                 /* if the RR is deleted twice or added twice, then we
4551                  * softfail, and continue with the rest of the IXFR, so
4552                  * that we serve something fairly nice during the refetch */
4553                 if(verbosity>=7) log_rrlist_position((delmode?"del":"add"),
4554                         rr_chunk, rr_dname, rr_type, rr_counter);
4555                 if(delmode) {
4556                         /* delete this RR */
4557                         int nonexist = 0;
4558                         if(!az_remove_rr_decompress(z, rr_chunk->data,
4559                                 rr_chunk->len, scratch_buffer, rr_dname,
4560                                 rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4561                                 &nonexist)) {
4562                                 /* failed, malloc error or so */
4563                                 return 0;
4564                         }
4565                         if(nonexist) {
4566                                 /* it was removal of a nonexisting RR */
4567                                 if(verbosity>=4) log_rrlist_position(
4568                                         "IXFR error nonexistent RR",
4569                                         rr_chunk, rr_dname, rr_type, rr_counter);
4570                                 softfail = 1;
4571                         }
4572                 } else if(rr_counter != 0) {
4573                         /* skip first SOA RR for addition, it is added in
4574                          * the addition part near the end of the ixfr, when
4575                          * that serial is seen the second time. */
4576                         int duplicate = 0;
4577                         /* add this RR */
4578                         if(!az_insert_rr_decompress(z, rr_chunk->data,
4579                                 rr_chunk->len, scratch_buffer, rr_dname,
4580                                 rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4581                                 &duplicate)) {
4582                                 /* failed, malloc error or so */
4583                                 return 0;
4584                         }
4585                         if(duplicate) {
4586                                 /* it was a duplicate */
4587                                 if(verbosity>=4) log_rrlist_position(
4588                                         "IXFR error duplicate RR",
4589                                         rr_chunk, rr_dname, rr_type, rr_counter);
4590                                 softfail = 1;
4591                         }
4592                 }
4593
4594                 rr_counter++;
4595                 chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4596         }
4597         if(softfail) {
4598                 verbose(VERB_ALGO, "IXFR did not apply cleanly, fetching full zone");
4599                 return 0;
4600         }
4601         return 1;
4602 }
4603
4604 /** apply AXFR to zone in memory. z is locked. false on failure(mallocfail) */
4605 static int
4606 apply_axfr(struct auth_xfer* xfr, struct auth_zone* z,
4607         struct sldns_buffer* scratch_buffer)
4608 {
4609         struct auth_chunk* rr_chunk;
4610         int rr_num;
4611         size_t rr_pos;
4612         uint8_t* rr_dname, *rr_rdata;
4613         uint16_t rr_type, rr_class, rr_rdlen;
4614         uint32_t rr_ttl;
4615         uint32_t serial = 0;
4616         size_t rr_nextpos;
4617         size_t rr_counter = 0;
4618         int have_end_soa = 0;
4619
4620         /* clear the data tree */
4621         traverse_postorder(&z->data, auth_data_del, NULL);
4622         rbtree_init(&z->data, &auth_data_cmp);
4623         xfr->have_zone = 0;
4624         xfr->serial = 0;
4625
4626         /* insert all RRs in to the zone */
4627         /* insert the SOA only once, skip the last one */
4628         /* start RR iterator over chunklist of packets */
4629         chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4630         while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4631                 if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4632                         &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4633                         &rr_rdata, &rr_nextpos)) {
4634                         /* failed to parse RR */
4635                         return 0;
4636                 }
4637                 if(verbosity>=7) log_rrlist_position("apply_axfr",
4638                         rr_chunk, rr_dname, rr_type, rr_counter);
4639                 if(rr_type == LDNS_RR_TYPE_SOA) {
4640                         if(rr_counter != 0) {
4641                                 /* end of the axfr */
4642                                 have_end_soa = 1;
4643                                 break;
4644                         }
4645                         if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4646                         serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4647                 }
4648
4649                 /* add this RR */
4650                 if(!az_insert_rr_decompress(z, rr_chunk->data, rr_chunk->len,
4651                         scratch_buffer, rr_dname, rr_type, rr_class, rr_ttl,
4652                         rr_rdata, rr_rdlen, NULL)) {
4653                         /* failed, malloc error or so */
4654                         return 0;
4655                 }
4656
4657                 rr_counter++;
4658                 chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4659         }
4660         if(!have_end_soa) {
4661                 log_err("no end SOA record for AXFR");
4662                 return 0;
4663         }
4664
4665         xfr->serial = serial;
4666         xfr->have_zone = 1;
4667         return 1;
4668 }
4669
4670 /** apply HTTP to zone in memory. z is locked. false on failure(mallocfail) */
4671 static int
4672 apply_http(struct auth_xfer* xfr, struct auth_zone* z,
4673         struct sldns_buffer* scratch_buffer)
4674 {
4675         /* parse data in chunks */
4676         /* parse RR's and read into memory. ignore $INCLUDE from the
4677          * downloaded file*/
4678         struct sldns_file_parse_state pstate;
4679         struct auth_chunk* chunk;
4680         size_t chunk_pos;
4681         memset(&pstate, 0, sizeof(pstate));
4682         pstate.default_ttl = 3600;
4683         if(xfr->namelen < sizeof(pstate.origin)) {
4684                 pstate.origin_len = xfr->namelen;
4685                 memmove(pstate.origin, xfr->name, xfr->namelen);
4686         }
4687
4688         if(verbosity >= VERB_ALGO)
4689                 verbose(VERB_ALGO, "http download %s of size %d",
4690                 xfr->task_transfer->master->file,
4691                 (int)chunklist_sum(xfr->task_transfer->chunks_first));
4692         if(xfr->task_transfer->chunks_first && verbosity >= VERB_ALGO) {
4693                 char preview[1024];
4694                 if(xfr->task_transfer->chunks_first->len+1 > sizeof(preview)) {
4695                         memmove(preview, xfr->task_transfer->chunks_first->data,
4696                                 sizeof(preview)-1);
4697                         preview[sizeof(preview)-1]=0;
4698                 } else {
4699                         memmove(preview, xfr->task_transfer->chunks_first->data,
4700                                 xfr->task_transfer->chunks_first->len);
4701                         preview[xfr->task_transfer->chunks_first->len]=0;
4702                 }
4703                 log_info("auth zone http downloaded content preview: %s",
4704                         preview);
4705         }
4706
4707         /* perhaps a little syntax check before we try to apply the data? */
4708         if(!http_zonefile_syntax_check(xfr, scratch_buffer)) {
4709                 log_err("http download %s/%s does not contain a zonefile, "
4710                         "but got '%s'", xfr->task_transfer->master->host,
4711                         xfr->task_transfer->master->file,
4712                         sldns_buffer_begin(scratch_buffer));
4713                 return 0;
4714         }
4715
4716         /* clear the data tree */
4717         traverse_postorder(&z->data, auth_data_del, NULL);
4718         rbtree_init(&z->data, &auth_data_cmp);
4719         xfr->have_zone = 0;
4720         xfr->serial = 0;
4721
4722         chunk = xfr->task_transfer->chunks_first;
4723         chunk_pos = 0;
4724         pstate.lineno = 0;
4725         while(chunkline_get_line_collated(&chunk, &chunk_pos, scratch_buffer)) {
4726                 /* process this line */
4727                 pstate.lineno++;
4728                 chunkline_newline_removal(scratch_buffer);
4729                 if(chunkline_is_comment_line_or_empty(scratch_buffer)) {
4730                         continue;
4731                 }
4732                 /* parse line and add RR */
4733                 if(http_parse_origin(scratch_buffer, &pstate)) {
4734                         continue; /* $ORIGIN has been handled */
4735                 }
4736                 if(http_parse_ttl(scratch_buffer, &pstate)) {
4737                         continue; /* $TTL has been handled */
4738                 }
4739                 if(!http_parse_add_rr(xfr, z, scratch_buffer, &pstate)) {
4740                         verbose(VERB_ALGO, "error parsing line [%s:%d] %s",
4741                                 xfr->task_transfer->master->file,
4742                                 pstate.lineno,
4743                                 sldns_buffer_begin(scratch_buffer));
4744                         return 0;
4745                 }
4746         }
4747         return 1;
4748 }
4749
4750 /** write http chunks to zonefile to create downloaded file */
4751 static int
4752 auth_zone_write_chunks(struct auth_xfer* xfr, const char* fname)
4753 {
4754         FILE* out;
4755         struct auth_chunk* p;
4756         out = fopen(fname, "w");
4757         if(!out) {
4758                 log_err("could not open %s: %s", fname, strerror(errno));
4759                 return 0;
4760         }
4761         for(p = xfr->task_transfer->chunks_first; p ; p = p->next) {
4762                 if(!write_out(out, (char*)p->data, p->len)) {
4763                         log_err("could not write http download to %s", fname);
4764                         fclose(out);
4765                         return 0;
4766                 }
4767         }
4768         fclose(out);
4769         return 1;
4770 }
4771
4772 /** write to zonefile after zone has been updated */
4773 static void
4774 xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
4775 {
4776         struct auth_zone* z;
4777         char tmpfile[1024];
4778         lock_basic_unlock(&xfr->lock);
4779
4780         /* get lock again, so it is a readlock and concurrently queries
4781          * can be answered */
4782         lock_rw_rdlock(&env->auth_zones->lock);
4783         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
4784                 xfr->dclass);
4785         if(!z) {
4786                 lock_rw_unlock(&env->auth_zones->lock);
4787                 /* the zone is gone, ignore xfr results */
4788                 lock_basic_lock(&xfr->lock);
4789                 return;
4790         }
4791         lock_rw_rdlock(&z->lock);
4792         lock_basic_lock(&xfr->lock);
4793         lock_rw_unlock(&env->auth_zones->lock);
4794
4795         if(z->zonefile == NULL) {
4796                 lock_rw_unlock(&z->lock);
4797                 /* no write needed, no zonefile set */
4798                 return;
4799         }
4800
4801         /* write to tempfile first */
4802         if((size_t)strlen(z->zonefile) + 16 > sizeof(tmpfile)) {
4803                 verbose(VERB_ALGO, "tmpfilename too long, cannot update "
4804                         " zonefile %s", z->zonefile);
4805                 lock_rw_unlock(&z->lock);
4806                 return;
4807         }
4808         snprintf(tmpfile, sizeof(tmpfile), "%s.tmp%u", z->zonefile,
4809                 (unsigned)getpid());
4810         if(xfr->task_transfer->master->http) {
4811                 /* use the stored chunk list to write them */
4812                 if(!auth_zone_write_chunks(xfr, tmpfile)) {
4813                         unlink(tmpfile);
4814                         lock_rw_unlock(&z->lock);
4815                 }
4816         } else if(!auth_zone_write_file(z, tmpfile)) {
4817                 unlink(tmpfile);
4818                 lock_rw_unlock(&z->lock);
4819                 return;
4820         }
4821         if(rename(tmpfile, z->zonefile) < 0) {
4822                 log_err("could not rename(%s, %s): %s", tmpfile, z->zonefile,
4823                         strerror(errno));
4824                 unlink(tmpfile);
4825                 lock_rw_unlock(&z->lock);
4826                 return;
4827         }
4828         lock_rw_unlock(&z->lock);
4829 }
4830
4831 /** process chunk list and update zone in memory,
4832  * return false if it did not work */
4833 static int
4834 xfr_process_chunk_list(struct auth_xfer* xfr, struct module_env* env,
4835         int* ixfr_fail)
4836 {
4837         struct auth_zone* z;
4838
4839         /* obtain locks and structures */
4840         /* release xfr lock, then, while holding az->lock grab both
4841          * z->lock and xfr->lock */
4842         lock_basic_unlock(&xfr->lock);
4843         lock_rw_rdlock(&env->auth_zones->lock);
4844         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
4845                 xfr->dclass);
4846         if(!z) {
4847                 lock_rw_unlock(&env->auth_zones->lock);
4848                 /* the zone is gone, ignore xfr results */
4849                 lock_basic_lock(&xfr->lock);
4850                 return 0;
4851         }
4852         lock_rw_wrlock(&z->lock);
4853         lock_basic_lock(&xfr->lock);
4854         lock_rw_unlock(&env->auth_zones->lock);
4855
4856         /* apply data */
4857         if(xfr->task_transfer->master->http) {
4858                 if(!apply_http(xfr, z, env->scratch_buffer)) {
4859                         lock_rw_unlock(&z->lock);
4860                         verbose(VERB_ALGO, "http from %s: could not store data",
4861                                 xfr->task_transfer->master->host);
4862                         return 0;
4863                 }
4864         } else if(xfr->task_transfer->on_ixfr &&
4865                 !xfr->task_transfer->on_ixfr_is_axfr) {
4866                 if(!apply_ixfr(xfr, z, env->scratch_buffer)) {
4867                         lock_rw_unlock(&z->lock);
4868                         verbose(VERB_ALGO, "xfr from %s: could not store IXFR"
4869                                 " data", xfr->task_transfer->master->host);
4870                         *ixfr_fail = 1;
4871                         return 0;
4872                 }
4873         } else {
4874                 if(!apply_axfr(xfr, z, env->scratch_buffer)) {
4875                         lock_rw_unlock(&z->lock);
4876                         verbose(VERB_ALGO, "xfr from %s: could not store AXFR"
4877                                 " data", xfr->task_transfer->master->host);
4878                         return 0;
4879                 }
4880         }
4881         xfr->zone_expired = 0;
4882         z->zone_expired = 0;
4883         if(!xfr_find_soa(z, xfr)) {
4884                 lock_rw_unlock(&z->lock);
4885                 verbose(VERB_ALGO, "xfr from %s: no SOA in zone after update"
4886                         " (or malformed RR)", xfr->task_transfer->master->host);
4887                 return 0;
4888         }
4889         if(xfr->have_zone)
4890                 xfr->lease_time = *env->now;
4891
4892         /* unlock */
4893         lock_rw_unlock(&z->lock);
4894
4895         if(verbosity >= VERB_QUERY && xfr->have_zone) {
4896                 char zname[256];
4897                 dname_str(xfr->name, zname);
4898                 verbose(VERB_QUERY, "auth zone %s updated to serial %u", zname,
4899                         (unsigned)xfr->serial);
4900         }
4901         /* see if we need to write to a zonefile */
4902         xfr_write_after_update(xfr, env);
4903         return 1;
4904 }
4905
4906 /** disown task_transfer.  caller must hold xfr.lock */
4907 static void
4908 xfr_transfer_disown(struct auth_xfer* xfr)
4909 {
4910         /* remove the commpoint */
4911         comm_point_delete(xfr->task_transfer->cp);
4912         xfr->task_transfer->cp = NULL;
4913         /* we don't own this item anymore */
4914         xfr->task_transfer->worker = NULL;
4915         xfr->task_transfer->env = NULL;
4916 }
4917
4918 /** lookup a host name for its addresses, if needed */
4919 static int
4920 xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
4921 {
4922         struct sockaddr_storage addr;
4923         socklen_t addrlen = 0;
4924         struct auth_master* master = xfr->task_transfer->lookup_target;
4925         struct query_info qinfo;
4926         uint16_t qflags = BIT_RD;
4927         uint8_t dname[LDNS_MAX_DOMAINLEN+1];
4928         struct edns_data edns;
4929         sldns_buffer* buf = env->scratch_buffer;
4930         if(!master) return 0;
4931         if(extstrtoaddr(master->host, &addr, &addrlen)) {
4932                 /* not needed, host is in IP addr format */
4933                 return 0;
4934         }
4935         if(master->allow_notify)
4936                 return 0; /* allow-notifies are not transferred from, no
4937                 lookup is needed */
4938
4939         /* use mesh_new_callback to probe for non-addr hosts,
4940          * and then wait for them to be looked up (in cache, or query) */
4941         qinfo.qname_len = sizeof(dname);
4942         if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
4943                 != 0) {
4944                 log_err("cannot parse host name of master %s", master->host);
4945                 return 0;
4946         }
4947         qinfo.qname = dname;
4948         qinfo.qclass = xfr->dclass;
4949         qinfo.qtype = LDNS_RR_TYPE_A;
4950         if(xfr->task_transfer->lookup_aaaa)
4951                 qinfo.qtype = LDNS_RR_TYPE_AAAA;
4952         qinfo.local_alias = NULL;
4953         if(verbosity >= VERB_ALGO) {
4954                 char buf[512];
4955                 char buf2[LDNS_MAX_DOMAINLEN+1];
4956                 dname_str(xfr->name, buf2);
4957                 snprintf(buf, sizeof(buf), "auth zone %s: master lookup"
4958                         " for task_transfer", buf2);
4959                 log_query_info(VERB_ALGO, buf, &qinfo);
4960         }
4961         edns.edns_present = 1;
4962         edns.ext_rcode = 0;
4963         edns.edns_version = 0;
4964         edns.bits = EDNS_DO;
4965         edns.opt_list = NULL;
4966         if(sldns_buffer_capacity(buf) < 65535)
4967                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
4968         else    edns.udp_size = 65535;
4969
4970         /* unlock xfr during mesh_new_callback() because the callback can be
4971          * called straight away */
4972         lock_basic_unlock(&xfr->lock);
4973         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
4974                 &auth_xfer_transfer_lookup_callback, xfr)) {
4975                 lock_basic_lock(&xfr->lock);
4976                 log_err("out of memory lookup up master %s", master->host);
4977                 return 0;
4978         }
4979         lock_basic_lock(&xfr->lock);
4980         return 1;
4981 }
4982
4983 /** initiate TCP to the target and fetch zone.
4984  * returns true if that was successfully started, and timeout setup. */
4985 static int
4986 xfr_transfer_init_fetch(struct auth_xfer* xfr, struct module_env* env)
4987 {
4988         struct sockaddr_storage addr;
4989         socklen_t addrlen = 0;
4990         struct auth_master* master = xfr->task_transfer->master;
4991         if(!master) return 0;
4992         if(master->allow_notify) return 0; /* only for notify */
4993
4994         /* get master addr */
4995         if(xfr->task_transfer->scan_addr) {
4996                 addrlen = xfr->task_transfer->scan_addr->addrlen;
4997                 memmove(&addr, &xfr->task_transfer->scan_addr->addr, addrlen);
4998         } else {
4999                 if(!extstrtoaddr(master->host, &addr, &addrlen)) {
5000                         /* the ones that are not in addr format are supposed
5001                          * to be looked up.  The lookup has failed however,
5002                          * so skip them */
5003                         char zname[255+1];
5004                         dname_str(xfr->name, zname);
5005                         log_err("%s: failed lookup, cannot transfer from master %s",
5006                                 zname, master->host);
5007                         return 0;
5008                 }
5009         }
5010
5011         /* remove previous TCP connection (if any) */
5012         if(xfr->task_transfer->cp) {
5013                 comm_point_delete(xfr->task_transfer->cp);
5014                 xfr->task_transfer->cp = NULL;
5015         }
5016
5017         if(master->http) {
5018                 /* perform http fetch */
5019                 /* store http port number into sockaddr,
5020                  * unless someone used unbound's host@port notation */
5021                 if(strchr(master->host, '@') == NULL)
5022                         sockaddr_store_port(&addr, addrlen, master->port);
5023                 xfr->task_transfer->cp = outnet_comm_point_for_http(
5024                         env->outnet, auth_xfer_transfer_http_callback, xfr,
5025                         &addr, addrlen, AUTH_TRANSFER_TIMEOUT, master->ssl,
5026                         master->host, master->file);
5027                 if(!xfr->task_transfer->cp) {
5028                         char zname[255+1];
5029                         dname_str(xfr->name, zname);
5030                         verbose(VERB_ALGO, "cannot create http cp "
5031                                 "connection for %s to %s", zname,
5032                                 master->host);
5033                         return 0;
5034                 }
5035                 return 1;
5036         }
5037
5038         /* perform AXFR/IXFR */
5039         /* set the packet to be written */
5040         /* create new ID */
5041         xfr->task_transfer->id = (uint16_t)(ub_random(env->rnd)&0xffff);
5042         xfr_create_ixfr_packet(xfr, env->scratch_buffer,
5043                 xfr->task_transfer->id, master);
5044
5045         /* connect on fd */
5046         xfr->task_transfer->cp = outnet_comm_point_for_tcp(env->outnet,
5047                 auth_xfer_transfer_tcp_callback, xfr, &addr, addrlen,
5048                 env->scratch_buffer, AUTH_TRANSFER_TIMEOUT);
5049         if(!xfr->task_transfer->cp) {
5050                 char zname[255+1];
5051                 dname_str(xfr->name, zname);
5052                 verbose(VERB_ALGO, "cannot create tcp cp connection for "
5053                         "xfr %s to %s", zname, master->host);
5054                 return 0;
5055         }
5056         return 1;
5057 }
5058
5059 /** perform next lookup, next transfer TCP, or end and resume wait time task */
5060 static void
5061 xfr_transfer_nexttarget_or_end(struct auth_xfer* xfr, struct module_env* env)
5062 {
5063         log_assert(xfr->task_transfer->worker == env->worker);
5064
5065         /* are we performing lookups? */
5066         while(xfr->task_transfer->lookup_target) {
5067                 if(xfr_transfer_lookup_host(xfr, env)) {
5068                         /* wait for lookup to finish,
5069                          * note that the hostname may be in unbound's cache
5070                          * and we may then get an instant cache response,
5071                          * and that calls the callback just like a full
5072                          * lookup and lookup failures also call callback */
5073                         lock_basic_unlock(&xfr->lock);
5074                         return;
5075                 }
5076                 xfr_transfer_move_to_next_lookup(xfr, env);
5077         }
5078
5079         /* initiate TCP and fetch the zone from the master */
5080         /* and set timeout on it */
5081         while(!xfr_transfer_end_of_list(xfr)) {
5082                 xfr->task_transfer->master = xfr_transfer_current_master(xfr);
5083                 if(xfr_transfer_init_fetch(xfr, env)) {
5084                         /* successfully started, wait for callback */
5085                         lock_basic_unlock(&xfr->lock);
5086                         return;
5087                 }
5088                 /* failed to fetch, next master */
5089                 xfr_transfer_nextmaster(xfr);
5090         }
5091
5092         /* we failed to fetch the zone, move to wait task
5093          * use the shorter retry timeout */
5094         xfr_transfer_disown(xfr);
5095
5096         /* pick up the nextprobe task and wait */
5097         if(xfr->task_nextprobe->worker == NULL)
5098                 xfr_set_timeout(xfr, env, 1, 0);
5099         lock_basic_unlock(&xfr->lock);
5100 }
5101
5102 /** add addrs from A or AAAA rrset to the master */
5103 static void
5104 xfr_master_add_addrs(struct auth_master* m, struct ub_packed_rrset_key* rrset,
5105         uint16_t rrtype)
5106 {
5107         size_t i;
5108         struct packed_rrset_data* data;
5109         if(!m || !rrset) return;
5110         if(rrtype != LDNS_RR_TYPE_A && rrtype != LDNS_RR_TYPE_AAAA)
5111                 return;
5112         data = (struct packed_rrset_data*)rrset->entry.data;
5113         for(i=0; i<data->count; i++) {
5114                 struct auth_addr* a;
5115                 size_t len = data->rr_len[i] - 2;
5116                 uint8_t* rdata = data->rr_data[i]+2;
5117                 if(rrtype == LDNS_RR_TYPE_A && len != INET_SIZE)
5118                         continue; /* wrong length for A */
5119                 if(rrtype == LDNS_RR_TYPE_AAAA && len != INET6_SIZE)
5120                         continue; /* wrong length for AAAA */
5121
5122                 /* add and alloc it */
5123                 a = (struct auth_addr*)calloc(1, sizeof(*a));
5124                 if(!a) {
5125                         log_err("out of memory");
5126                         return;
5127                 }
5128                 if(rrtype == LDNS_RR_TYPE_A) {
5129                         struct sockaddr_in* sa;
5130                         a->addrlen = (socklen_t)sizeof(*sa);
5131                         sa = (struct sockaddr_in*)&a->addr;
5132                         sa->sin_family = AF_INET;
5133                         sa->sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5134                         memmove(&sa->sin_addr, rdata, INET_SIZE);
5135                 } else {
5136                         struct sockaddr_in6* sa;
5137                         a->addrlen = (socklen_t)sizeof(*sa);
5138                         sa = (struct sockaddr_in6*)&a->addr;
5139                         sa->sin6_family = AF_INET6;
5140                         sa->sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5141                         memmove(&sa->sin6_addr, rdata, INET6_SIZE);
5142                 }
5143                 if(verbosity >= VERB_ALGO) {
5144                         char s[64];
5145                         addr_to_str(&a->addr, a->addrlen, s, sizeof(s));
5146                         verbose(VERB_ALGO, "auth host %s lookup %s",
5147                                 m->host, s);
5148                 }
5149                 /* append to list */
5150                 a->next = m->list;
5151                 m->list = a;
5152         }
5153 }
5154
5155 /** callback for task_transfer lookup of host name, of A or AAAA */
5156 void auth_xfer_transfer_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
5157         enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
5158         int ATTR_UNUSED(was_ratelimited))
5159 {
5160         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5161         struct module_env* env;
5162         log_assert(xfr->task_transfer);
5163         lock_basic_lock(&xfr->lock);
5164         env = xfr->task_transfer->env;
5165         if(env->outnet->want_to_quit) {
5166                 lock_basic_unlock(&xfr->lock);
5167                 return; /* stop on quit */
5168         }
5169
5170         /* process result */
5171         if(rcode == LDNS_RCODE_NOERROR) {
5172                 uint16_t wanted_qtype = LDNS_RR_TYPE_A;
5173                 struct regional* temp = env->scratch;
5174                 struct query_info rq;
5175                 struct reply_info* rep;
5176                 if(xfr->task_transfer->lookup_aaaa)
5177                         wanted_qtype = LDNS_RR_TYPE_AAAA;
5178                 memset(&rq, 0, sizeof(rq));
5179                 rep = parse_reply_in_temp_region(buf, temp, &rq);
5180                 if(rep && rq.qtype == wanted_qtype &&
5181                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
5182                         /* parsed successfully */
5183                         struct ub_packed_rrset_key* answer =
5184                                 reply_find_answer_rrset(&rq, rep);
5185                         if(answer) {
5186                                 xfr_master_add_addrs(xfr->task_transfer->
5187                                         lookup_target, answer, wanted_qtype);
5188                         }
5189                 }
5190         }
5191         if(xfr->task_transfer->lookup_target->list &&
5192                 xfr->task_transfer->lookup_target == xfr_transfer_current_master(xfr))
5193                 xfr->task_transfer->scan_addr = xfr->task_transfer->lookup_target->list;
5194
5195         /* move to lookup AAAA after A lookup, move to next hostname lookup,
5196          * or move to fetch the zone, or, if nothing to do, end task_transfer */
5197         xfr_transfer_move_to_next_lookup(xfr, env);
5198         xfr_transfer_nexttarget_or_end(xfr, env);
5199 }
5200
5201 /** check if xfer (AXFR or IXFR) packet is OK.
5202  * return false if we lost connection (SERVFAIL, or unreadable).
5203  * return false if we need to move from IXFR to AXFR, with gonextonfail
5204  *      set to false, so the same master is tried again, but with AXFR.
5205  * return true if fine to link into data.
5206  * return true with transferdone=true when the transfer has ended.
5207  */
5208 static int
5209 check_xfer_packet(sldns_buffer* pkt, struct auth_xfer* xfr,
5210         int* gonextonfail, int* transferdone)
5211 {
5212         uint8_t* wire = sldns_buffer_begin(pkt);
5213         int i;
5214         if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE) {
5215                 verbose(VERB_ALGO, "xfr to %s failed, packet too small",
5216                         xfr->task_transfer->master->host);
5217                 return 0;
5218         }
5219         if(!LDNS_QR_WIRE(wire)) {
5220                 verbose(VERB_ALGO, "xfr to %s failed, packet has no QR flag",
5221                         xfr->task_transfer->master->host);
5222                 return 0;
5223         }
5224         if(LDNS_TC_WIRE(wire)) {
5225                 verbose(VERB_ALGO, "xfr to %s failed, packet has TC flag",
5226                         xfr->task_transfer->master->host);
5227                 return 0;
5228         }
5229         /* check ID */
5230         if(LDNS_ID_WIRE(wire) != xfr->task_transfer->id) {
5231                 verbose(VERB_ALGO, "xfr to %s failed, packet wrong ID",
5232                         xfr->task_transfer->master->host);
5233                 return 0;
5234         }
5235         if(LDNS_RCODE_WIRE(wire) != LDNS_RCODE_NOERROR) {
5236                 char rcode[32];
5237                 sldns_wire2str_rcode_buf((int)LDNS_RCODE_WIRE(wire), rcode,
5238                         sizeof(rcode));
5239                 /* if we are doing IXFR, check for fallback */
5240                 if(xfr->task_transfer->on_ixfr) {
5241                         if(LDNS_RCODE_WIRE(wire) == LDNS_RCODE_NOTIMPL ||
5242                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_SERVFAIL ||
5243                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_REFUSED ||
5244                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_FORMERR) {
5245                                 verbose(VERB_ALGO, "xfr to %s, fallback "
5246                                         "from IXFR to AXFR (with rcode %s)",
5247                                         xfr->task_transfer->master->host,
5248                                         rcode);
5249                                 xfr->task_transfer->ixfr_fail = 1;
5250                                 *gonextonfail = 0;
5251                                 return 0;
5252                         }
5253                 }
5254                 verbose(VERB_ALGO, "xfr to %s failed, packet with rcode %s",
5255                         xfr->task_transfer->master->host, rcode);
5256                 return 0;
5257         }
5258         if(LDNS_OPCODE_WIRE(wire) != LDNS_PACKET_QUERY) {
5259                 verbose(VERB_ALGO, "xfr to %s failed, packet with bad opcode",
5260                         xfr->task_transfer->master->host);
5261                 return 0;
5262         }
5263         if(LDNS_QDCOUNT(wire) > 1) {
5264                 verbose(VERB_ALGO, "xfr to %s failed, packet has qdcount %d",
5265                         xfr->task_transfer->master->host,
5266                         (int)LDNS_QDCOUNT(wire));
5267                 return 0;
5268         }
5269
5270         /* check qname */
5271         sldns_buffer_set_position(pkt, LDNS_HEADER_SIZE);
5272         for(i=0; i<(int)LDNS_QDCOUNT(wire); i++) {
5273                 size_t pos = sldns_buffer_position(pkt);
5274                 uint16_t qtype, qclass;
5275                 if(pkt_dname_len(pkt) == 0) {
5276                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5277                                 "malformed dname",
5278                                 xfr->task_transfer->master->host);
5279                         return 0;
5280                 }
5281                 if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5282                         xfr->name) != 0) {
5283                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5284                                 "wrong qname",
5285                                 xfr->task_transfer->master->host);
5286                         return 0;
5287                 }
5288                 if(sldns_buffer_remaining(pkt) < 4) {
5289                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5290                                 "truncated query RR",
5291                                 xfr->task_transfer->master->host);
5292                         return 0;
5293                 }
5294                 qtype = sldns_buffer_read_u16(pkt);
5295                 qclass = sldns_buffer_read_u16(pkt);
5296                 if(qclass != xfr->dclass) {
5297                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5298                                 "wrong qclass",
5299                                 xfr->task_transfer->master->host);
5300                         return 0;
5301                 }
5302                 if(xfr->task_transfer->on_ixfr) {
5303                         if(qtype != LDNS_RR_TYPE_IXFR) {
5304                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5305                                         "with wrong qtype, expected IXFR",
5306                                 xfr->task_transfer->master->host);
5307                                 return 0;
5308                         }
5309                 } else {
5310                         if(qtype != LDNS_RR_TYPE_AXFR) {
5311                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5312                                         "with wrong qtype, expected AXFR",
5313                                 xfr->task_transfer->master->host);
5314                                 return 0;
5315                         }
5316                 }
5317         }
5318
5319         /* check parse of RRs in packet, store first SOA serial
5320          * to be able to detect last SOA (with that serial) to see if done */
5321         /* also check for IXFR 'zone up to date' reply */
5322         for(i=0; i<(int)LDNS_ANCOUNT(wire); i++) {
5323                 size_t pos = sldns_buffer_position(pkt);
5324                 uint16_t tp, rdlen;
5325                 if(pkt_dname_len(pkt) == 0) {
5326                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5327                                 "malformed dname in answer section",
5328                                 xfr->task_transfer->master->host);
5329                         return 0;
5330                 }
5331                 if(sldns_buffer_remaining(pkt) < 10) {
5332                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5333                                 "truncated RR",
5334                                 xfr->task_transfer->master->host);
5335                         return 0;
5336                 }
5337                 tp = sldns_buffer_read_u16(pkt);
5338                 (void)sldns_buffer_read_u16(pkt); /* class */
5339                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5340                 rdlen = sldns_buffer_read_u16(pkt);
5341                 if(sldns_buffer_remaining(pkt) < rdlen) {
5342                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5343                                 "truncated RR rdata",
5344                                 xfr->task_transfer->master->host);
5345                         return 0;
5346                 }
5347
5348                 /* RR parses (haven't checked rdata itself), now look at
5349                  * SOA records to see serial number */
5350                 if(xfr->task_transfer->rr_scan_num == 0 &&
5351                         tp != LDNS_RR_TYPE_SOA) {
5352                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5353                                 "malformed zone transfer, no start SOA",
5354                                 xfr->task_transfer->master->host);
5355                         return 0;
5356                 }
5357                 if(xfr->task_transfer->rr_scan_num == 1 &&
5358                         tp != LDNS_RR_TYPE_SOA) {
5359                         /* second RR is not a SOA record, this is not an IXFR
5360                          * the master is replying with an AXFR */
5361                         xfr->task_transfer->on_ixfr_is_axfr = 1;
5362                 }
5363                 if(tp == LDNS_RR_TYPE_SOA) {
5364                         uint32_t serial;
5365                         if(rdlen < 22) {
5366                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5367                                         "with SOA with malformed rdata",
5368                                         xfr->task_transfer->master->host);
5369                                 return 0;
5370                         }
5371                         if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5372                                 xfr->name) != 0) {
5373                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5374                                         "with SOA with wrong dname",
5375                                         xfr->task_transfer->master->host);
5376                                 return 0;
5377                         }
5378
5379                         /* read serial number of SOA */
5380                         serial = sldns_buffer_read_u32_at(pkt,
5381                                 sldns_buffer_position(pkt)+rdlen-20);
5382
5383                         /* check for IXFR 'zone has SOA x' reply */
5384                         if(xfr->task_transfer->on_ixfr &&
5385                                 xfr->task_transfer->rr_scan_num == 0 &&
5386                                 LDNS_ANCOUNT(wire)==1) {
5387                                 verbose(VERB_ALGO, "xfr to %s ended, "
5388                                         "IXFR reply that zone has serial %u",
5389                                         xfr->task_transfer->master->host,
5390                                         (unsigned)serial);
5391                                 return 0;
5392                         }
5393
5394                         /* if first SOA, store serial number */
5395                         if(xfr->task_transfer->got_xfr_serial == 0) {
5396                                 xfr->task_transfer->got_xfr_serial = 1;
5397                                 xfr->task_transfer->incoming_xfr_serial =
5398                                         serial;
5399                                 verbose(VERB_ALGO, "xfr %s: contains "
5400                                         "SOA serial %u",
5401                                         xfr->task_transfer->master->host,
5402                                         (unsigned)serial);
5403                         /* see if end of AXFR */
5404                         } else if(!xfr->task_transfer->on_ixfr ||
5405                                 xfr->task_transfer->on_ixfr_is_axfr) {
5406                                 /* second SOA with serial is the end
5407                                  * for AXFR */
5408                                 *transferdone = 1;
5409                                 verbose(VERB_ALGO, "xfr %s: last AXFR packet",
5410                                         xfr->task_transfer->master->host);
5411                         /* for IXFR, count SOA records with that serial */
5412                         } else if(xfr->task_transfer->incoming_xfr_serial ==
5413                                 serial && xfr->task_transfer->got_xfr_serial
5414                                 == 1) {
5415                                 xfr->task_transfer->got_xfr_serial++;
5416                         /* if not first soa, if serial==firstserial, the
5417                          * third time we are at the end, for IXFR */
5418                         } else if(xfr->task_transfer->incoming_xfr_serial ==
5419                                 serial && xfr->task_transfer->got_xfr_serial
5420                                 == 2) {
5421                                 verbose(VERB_ALGO, "xfr %s: last IXFR packet",
5422                                         xfr->task_transfer->master->host);
5423                                 *transferdone = 1;
5424                                 /* continue parse check, if that succeeds,
5425                                  * transfer is done */
5426                         }
5427                 }
5428                 xfr->task_transfer->rr_scan_num++;
5429
5430                 /* skip over RR rdata to go to the next RR */
5431                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5432         }
5433
5434         /* check authority section */
5435         /* we skip over the RRs checking packet format */
5436         for(i=0; i<(int)LDNS_NSCOUNT(wire); i++) {
5437                 uint16_t rdlen;
5438                 if(pkt_dname_len(pkt) == 0) {
5439                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5440                                 "malformed dname in authority section",
5441                                 xfr->task_transfer->master->host);
5442                         return 0;
5443                 }
5444                 if(sldns_buffer_remaining(pkt) < 10) {
5445                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5446                                 "truncated RR",
5447                                 xfr->task_transfer->master->host);
5448                         return 0;
5449                 }
5450                 (void)sldns_buffer_read_u16(pkt); /* type */
5451                 (void)sldns_buffer_read_u16(pkt); /* class */
5452                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5453                 rdlen = sldns_buffer_read_u16(pkt);
5454                 if(sldns_buffer_remaining(pkt) < rdlen) {
5455                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5456                                 "truncated RR rdata",
5457                                 xfr->task_transfer->master->host);
5458                         return 0;
5459                 }
5460                 /* skip over RR rdata to go to the next RR */
5461                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5462         }
5463
5464         /* check additional section */
5465         for(i=0; i<(int)LDNS_ARCOUNT(wire); i++) {
5466                 uint16_t rdlen;
5467                 if(pkt_dname_len(pkt) == 0) {
5468                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5469                                 "malformed dname in additional section",
5470                                 xfr->task_transfer->master->host);
5471                         return 0;
5472                 }
5473                 if(sldns_buffer_remaining(pkt) < 10) {
5474                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5475                                 "truncated RR",
5476                                 xfr->task_transfer->master->host);
5477                         return 0;
5478                 }
5479                 (void)sldns_buffer_read_u16(pkt); /* type */
5480                 (void)sldns_buffer_read_u16(pkt); /* class */
5481                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5482                 rdlen = sldns_buffer_read_u16(pkt);
5483                 if(sldns_buffer_remaining(pkt) < rdlen) {
5484                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5485                                 "truncated RR rdata",
5486                                 xfr->task_transfer->master->host);
5487                         return 0;
5488                 }
5489                 /* skip over RR rdata to go to the next RR */
5490                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5491         }
5492
5493         return 1;
5494 }
5495
5496 /** Link the data from this packet into the worklist of transferred data */
5497 static int
5498 xfer_link_data(sldns_buffer* pkt, struct auth_xfer* xfr)
5499 {
5500         /* alloc it */
5501         struct auth_chunk* e;
5502         e = (struct auth_chunk*)calloc(1, sizeof(*e));
5503         if(!e) return 0;
5504         e->next = NULL;
5505         e->len = sldns_buffer_limit(pkt);
5506         e->data = memdup(sldns_buffer_begin(pkt), e->len);
5507         if(!e->data) {
5508                 free(e);
5509                 return 0;
5510         }
5511
5512         /* alloc succeeded, link into list */
5513         if(!xfr->task_transfer->chunks_first)
5514                 xfr->task_transfer->chunks_first = e;
5515         if(xfr->task_transfer->chunks_last)
5516                 xfr->task_transfer->chunks_last->next = e;
5517         xfr->task_transfer->chunks_last = e;
5518         return 1;
5519 }
5520
5521 /** task transfer.  the list of data is complete. process it and if failed
5522  * move to next master, if succeeded, end the task transfer */
5523 static void
5524 process_list_end_transfer(struct auth_xfer* xfr, struct module_env* env)
5525 {
5526         int ixfr_fail = 0;
5527         if(xfr_process_chunk_list(xfr, env, &ixfr_fail)) {
5528                 /* it worked! */
5529                 auth_chunks_delete(xfr->task_transfer);
5530
5531                 /* we fetched the zone, move to wait task */
5532                 xfr_transfer_disown(xfr);
5533
5534                 if(xfr->notify_received && (!xfr->notify_has_serial ||
5535                         (xfr->notify_has_serial &&
5536                         xfr_serial_means_update(xfr, xfr->notify_serial)))) {
5537                         uint32_t sr = xfr->notify_serial;
5538                         int has_sr = xfr->notify_has_serial;
5539                         /* we received a notify while probe/transfer was
5540                          * in progress.  start a new probe and transfer */
5541                         xfr->notify_received = 0;
5542                         xfr->notify_has_serial = 0;
5543                         xfr->notify_serial = 0;
5544                         if(!xfr_start_probe(xfr, env, NULL)) {
5545                                 /* if we couldn't start it, already in
5546                                  * progress; restore notify serial,
5547                                  * while xfr still locked */
5548                                 xfr->notify_received = 1;
5549                                 xfr->notify_has_serial = has_sr;
5550                                 xfr->notify_serial = sr;
5551                                 lock_basic_unlock(&xfr->lock);
5552                         }
5553                         return;
5554                 } else {
5555                         /* pick up the nextprobe task and wait (normail wait time) */
5556                         if(xfr->task_nextprobe->worker == NULL)
5557                                 xfr_set_timeout(xfr, env, 0, 0);
5558                 }
5559                 lock_basic_unlock(&xfr->lock);
5560                 return;
5561         }
5562         /* processing failed */
5563         /* when done, delete data from list */
5564         auth_chunks_delete(xfr->task_transfer);
5565         if(ixfr_fail) {
5566                 xfr->task_transfer->ixfr_fail = 1;
5567         } else {
5568                 xfr_transfer_nextmaster(xfr);
5569         }
5570         xfr_transfer_nexttarget_or_end(xfr, env);
5571 }
5572
5573 /** callback for task_transfer tcp connections */
5574 int
5575 auth_xfer_transfer_tcp_callback(struct comm_point* c, void* arg, int err,
5576         struct comm_reply* ATTR_UNUSED(repinfo))
5577 {
5578         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5579         struct module_env* env;
5580         int gonextonfail = 1;
5581         int transferdone = 0;
5582         log_assert(xfr->task_transfer);
5583         lock_basic_lock(&xfr->lock);
5584         env = xfr->task_transfer->env;
5585         if(env->outnet->want_to_quit) {
5586                 lock_basic_unlock(&xfr->lock);
5587                 return 0; /* stop on quit */
5588         }
5589
5590         if(err != NETEVENT_NOERROR) {
5591                 /* connection failed, closed, or timeout */
5592                 /* stop this transfer, cleanup
5593                  * and continue task_transfer*/
5594                 verbose(VERB_ALGO, "xfr stopped, connection lost to %s",
5595                         xfr->task_transfer->master->host);
5596         failed:
5597                 /* delete transferred data from list */
5598                 auth_chunks_delete(xfr->task_transfer);
5599                 comm_point_delete(xfr->task_transfer->cp);
5600                 xfr->task_transfer->cp = NULL;
5601                 xfr_transfer_nextmaster(xfr);
5602                 xfr_transfer_nexttarget_or_end(xfr, env);
5603                 return 0;
5604         }
5605
5606         /* handle returned packet */
5607         /* if it fails, cleanup and end this transfer */
5608         /* if it needs to fallback from IXFR to AXFR, do that */
5609         if(!check_xfer_packet(c->buffer, xfr, &gonextonfail, &transferdone)) {
5610                 goto failed;
5611         }
5612         /* if it is good, link it into the list of data */
5613         /* if the link into list of data fails (malloc fail) cleanup and end */
5614         if(!xfer_link_data(c->buffer, xfr)) {
5615                 verbose(VERB_ALGO, "xfr stopped to %s, malloc failed",
5616                         xfr->task_transfer->master->host);
5617                 goto failed;
5618         }
5619         /* if the transfer is done now, disconnect and process the list */
5620         if(transferdone) {
5621                 comm_point_delete(xfr->task_transfer->cp);
5622                 xfr->task_transfer->cp = NULL;
5623                 process_list_end_transfer(xfr, env);
5624                 return 0;
5625         }
5626
5627         /* if we want to read more messages, setup the commpoint to read
5628          * a DNS packet, and the timeout */
5629         lock_basic_unlock(&xfr->lock);
5630         c->tcp_is_reading = 1;
5631         sldns_buffer_clear(c->buffer);
5632         comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
5633         return 0;
5634 }
5635
5636 /** callback for task_transfer http connections */
5637 int
5638 auth_xfer_transfer_http_callback(struct comm_point* c, void* arg, int err,
5639         struct comm_reply* repinfo)
5640 {
5641         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5642         struct module_env* env;
5643         log_assert(xfr->task_transfer);
5644         lock_basic_lock(&xfr->lock);
5645         env = xfr->task_transfer->env;
5646         if(env->outnet->want_to_quit) {
5647                 lock_basic_unlock(&xfr->lock);
5648                 return 0; /* stop on quit */
5649         }
5650         verbose(VERB_ALGO, "auth zone transfer http callback");
5651
5652         if(err != NETEVENT_NOERROR && err != NETEVENT_DONE) {
5653                 /* connection failed, closed, or timeout */
5654                 /* stop this transfer, cleanup
5655                  * and continue task_transfer*/
5656                 verbose(VERB_ALGO, "http stopped, connection lost to %s",
5657                         xfr->task_transfer->master->host);
5658         failed:
5659                 /* delete transferred data from list */
5660                 auth_chunks_delete(xfr->task_transfer);
5661                 if(repinfo) repinfo->c = NULL; /* signal cp deleted to
5662                                 the routine calling this callback */
5663                 comm_point_delete(xfr->task_transfer->cp);
5664                 xfr->task_transfer->cp = NULL;
5665                 xfr_transfer_nextmaster(xfr);
5666                 xfr_transfer_nexttarget_or_end(xfr, env);
5667                 return 0;
5668         }
5669
5670         /* if it is good, link it into the list of data */
5671         /* if the link into list of data fails (malloc fail) cleanup and end */
5672         if(sldns_buffer_limit(c->buffer) > 0) {
5673                 verbose(VERB_ALGO, "auth zone http queued up %d bytes",
5674                         (int)sldns_buffer_limit(c->buffer));
5675                 if(!xfer_link_data(c->buffer, xfr)) {
5676                         verbose(VERB_ALGO, "http stopped to %s, malloc failed",
5677                                 xfr->task_transfer->master->host);
5678                         goto failed;
5679                 }
5680         }
5681         /* if the transfer is done now, disconnect and process the list */
5682         if(err == NETEVENT_DONE) {
5683                 if(repinfo) repinfo->c = NULL; /* signal cp deleted to
5684                                 the routine calling this callback */
5685                 comm_point_delete(xfr->task_transfer->cp);
5686                 xfr->task_transfer->cp = NULL;
5687                 process_list_end_transfer(xfr, env);
5688                 return 0;
5689         }
5690
5691         /* if we want to read more messages, setup the commpoint to read
5692          * a DNS packet, and the timeout */
5693         lock_basic_unlock(&xfr->lock);
5694         c->tcp_is_reading = 1;
5695         sldns_buffer_clear(c->buffer);
5696         comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
5697         return 0;
5698 }
5699
5700
5701 /** start transfer task by this worker , xfr is locked. */
5702 static void
5703 xfr_start_transfer(struct auth_xfer* xfr, struct module_env* env,
5704         struct auth_master* master)
5705 {
5706         log_assert(xfr->task_transfer != NULL);
5707         log_assert(xfr->task_transfer->worker == NULL);
5708         log_assert(xfr->task_transfer->chunks_first == NULL);
5709         log_assert(xfr->task_transfer->chunks_last == NULL);
5710         xfr->task_transfer->worker = env->worker;
5711         xfr->task_transfer->env = env;
5712
5713         /* init transfer process */
5714         /* find that master in the transfer's list of masters? */
5715         xfr_transfer_start_list(xfr, master);
5716         /* start lookup for hostnames in transfer master list */
5717         xfr_transfer_start_lookups(xfr);
5718
5719         /* initiate TCP, and set timeout on it */
5720         xfr_transfer_nexttarget_or_end(xfr, env);
5721 }
5722
5723 /** disown task_probe.  caller must hold xfr.lock */
5724 static void
5725 xfr_probe_disown(struct auth_xfer* xfr)
5726 {
5727         /* remove timer (from this worker's event base) */
5728         comm_timer_delete(xfr->task_probe->timer);
5729         xfr->task_probe->timer = NULL;
5730         /* remove the commpoint */
5731         comm_point_delete(xfr->task_probe->cp);
5732         xfr->task_probe->cp = NULL;
5733         /* we don't own this item anymore */
5734         xfr->task_probe->worker = NULL;
5735         xfr->task_probe->env = NULL;
5736 }
5737
5738 /** send the UDP probe to the master, this is part of task_probe */
5739 static int
5740 xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
5741         int timeout)
5742 {
5743         struct sockaddr_storage addr;
5744         socklen_t addrlen = 0;
5745         struct timeval t;
5746         /* pick master */
5747         struct auth_master* master = xfr_probe_current_master(xfr);
5748         if(!master) return 0;
5749         if(master->allow_notify) return 0; /* only for notify */
5750         if(master->http) return 0; /* only masters get SOA UDP probe,
5751                 not urls, if those are in this list */
5752
5753         /* get master addr */
5754         if(xfr->task_probe->scan_addr) {
5755                 addrlen = xfr->task_probe->scan_addr->addrlen;
5756                 memmove(&addr, &xfr->task_probe->scan_addr->addr, addrlen);
5757         } else {
5758                 if(!extstrtoaddr(master->host, &addr, &addrlen)) {
5759                         /* the ones that are not in addr format are supposed
5760                          * to be looked up.  The lookup has failed however,
5761                          * so skip them */
5762                         char zname[255+1];
5763                         dname_str(xfr->name, zname);
5764                         log_err("%s: failed lookup, cannot probe to master %s",
5765                                 zname, master->host);
5766                         return 0;
5767                 }
5768         }
5769
5770         /* create packet */
5771         /* create new ID for new probes, but not on timeout retries,
5772          * this means we'll accept replies to previous retries to same ip */
5773         if(timeout == AUTH_PROBE_TIMEOUT)
5774                 xfr->task_probe->id = (uint16_t)(ub_random(env->rnd)&0xffff);
5775         xfr_create_soa_probe_packet(xfr, env->scratch_buffer,
5776                 xfr->task_probe->id);
5777         if(!xfr->task_probe->cp) {
5778                 xfr->task_probe->cp = outnet_comm_point_for_udp(env->outnet,
5779                         auth_xfer_probe_udp_callback, xfr, &addr, addrlen);
5780                 if(!xfr->task_probe->cp) {
5781                         char zname[255+1];
5782                         dname_str(xfr->name, zname);
5783                         verbose(VERB_ALGO, "cannot create udp cp for "
5784                                 "probe %s to %s", zname, master->host);
5785                         return 0;
5786                 }
5787         }
5788         if(!xfr->task_probe->timer) {
5789                 xfr->task_probe->timer = comm_timer_create(env->worker_base,
5790                         auth_xfer_probe_timer_callback, xfr);
5791                 if(!xfr->task_probe->timer) {
5792                         log_err("malloc failure");
5793                         return 0;
5794                 }
5795         }
5796
5797         /* send udp packet */
5798         if(!comm_point_send_udp_msg(xfr->task_probe->cp, env->scratch_buffer,
5799                 (struct sockaddr*)&addr, addrlen)) {
5800                 char zname[255+1];
5801                 dname_str(xfr->name, zname);
5802                 verbose(VERB_ALGO, "failed to send soa probe for %s to %s",
5803                         zname, master->host);
5804                 return 0;
5805         }
5806         xfr->task_probe->timeout = timeout;
5807 #ifndef S_SPLINT_S
5808         t.tv_sec = timeout/1000;
5809         t.tv_usec = (timeout%1000)*1000;
5810 #endif
5811         comm_timer_set(xfr->task_probe->timer, &t);
5812
5813         return 1;
5814 }
5815
5816 /** callback for task_probe timer */
5817 void
5818 auth_xfer_probe_timer_callback(void* arg)
5819 {
5820         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5821         struct module_env* env;
5822         log_assert(xfr->task_probe);
5823         lock_basic_lock(&xfr->lock);
5824         env = xfr->task_probe->env;
5825         if(env->outnet->want_to_quit) {
5826                 lock_basic_unlock(&xfr->lock);
5827                 return; /* stop on quit */
5828         }
5829
5830         if(xfr->task_probe->timeout <= AUTH_PROBE_TIMEOUT_STOP) {
5831                 /* try again with bigger timeout */
5832                 if(xfr_probe_send_probe(xfr, env, xfr->task_probe->timeout*2)) {
5833                         lock_basic_unlock(&xfr->lock);
5834                         return;
5835                 }
5836         }
5837         /* delete commpoint so a new one is created, with a fresh port nr */
5838         comm_point_delete(xfr->task_probe->cp);
5839         xfr->task_probe->cp = NULL;
5840
5841         /* too many timeouts (or fail to send), move to next or end */
5842         xfr_probe_nextmaster(xfr);
5843         xfr_probe_send_or_end(xfr, env);
5844 }
5845
5846 /** callback for task_probe udp packets */
5847 int
5848 auth_xfer_probe_udp_callback(struct comm_point* c, void* arg, int err,
5849         struct comm_reply* repinfo)
5850 {
5851         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5852         struct module_env* env;
5853         log_assert(xfr->task_probe);
5854         lock_basic_lock(&xfr->lock);
5855         env = xfr->task_probe->env;
5856         if(env->outnet->want_to_quit) {
5857                 lock_basic_unlock(&xfr->lock);
5858                 return 0; /* stop on quit */
5859         }
5860
5861         /* the comm_point_udp_callback is in a for loop for NUM_UDP_PER_SELECT
5862          * and we set rep.c=NULL to stop if from looking inside the commpoint*/
5863         repinfo->c = NULL;
5864         /* stop the timer */
5865         comm_timer_disable(xfr->task_probe->timer);
5866
5867         /* see if we got a packet and what that means */
5868         if(err == NETEVENT_NOERROR) {
5869                 uint32_t serial = 0;
5870                 if(check_packet_ok(c->buffer, LDNS_RR_TYPE_SOA, xfr,
5871                         &serial)) {
5872                         /* successful lookup */
5873                         if(verbosity >= VERB_ALGO) {
5874                                 char buf[256];
5875                                 dname_str(xfr->name, buf);
5876                                 verbose(VERB_ALGO, "auth zone %s: soa probe "
5877                                         "serial is %u", buf, (unsigned)serial);
5878                         }
5879                         /* see if this serial indicates that the zone has
5880                          * to be updated */
5881                         if(xfr_serial_means_update(xfr, serial)) {
5882                                 /* if updated, start the transfer task, if needed */
5883                                 verbose(VERB_ALGO, "auth_zone updated, start transfer");
5884                                 if(xfr->task_transfer->worker == NULL) {
5885                                         struct auth_master* master =
5886                                                 xfr_probe_current_master(xfr);
5887                                         /* if we have download URLs use them
5888                                          * in preference to this master we
5889                                          * just probed the SOA from */
5890                                         if(xfr->task_transfer->masters &&
5891                                                 xfr->task_transfer->masters->http)
5892                                                 master = NULL;
5893                                         xfr_probe_disown(xfr);
5894                                         xfr_start_transfer(xfr, env, master);
5895                                         return 0;
5896
5897                                 }
5898                                 /* other tasks are running, we don't do this anymore */
5899                                 xfr_probe_disown(xfr);
5900                                 lock_basic_unlock(&xfr->lock);
5901                                 /* return, we don't sent a reply to this udp packet,
5902                                  * and we setup the tasks to do next */
5903                                 return 0;
5904                         } else {
5905                                 verbose(VERB_ALGO, "auth_zone master reports unchanged soa serial");
5906                                 /* we if cannot find updates amongst the
5907                                  * masters, this means we then have a new lease
5908                                  * on the zone */
5909                                 xfr->task_probe->have_new_lease = 1;
5910                         }
5911                 } else {
5912                         if(verbosity >= VERB_ALGO) {
5913                                 char buf[256];
5914                                 dname_str(xfr->name, buf);
5915                                 verbose(VERB_ALGO, "auth zone %s: bad reply to soa probe", buf);
5916                         }
5917                 }
5918         } else {
5919                 if(verbosity >= VERB_ALGO) {
5920                         char buf[256];
5921                         dname_str(xfr->name, buf);
5922                         verbose(VERB_ALGO, "auth zone %s: soa probe failed", buf);
5923                 }
5924         }
5925
5926         /* failed lookup or not an update */
5927         /* delete commpoint so a new one is created, with a fresh port nr */
5928         comm_point_delete(xfr->task_probe->cp);
5929         xfr->task_probe->cp = NULL;
5930
5931         /* if the result was not a successfull probe, we need
5932          * to send the next one */
5933         xfr_probe_nextmaster(xfr);
5934         xfr_probe_send_or_end(xfr, env);
5935         return 0;
5936 }
5937
5938 /** lookup a host name for its addresses, if needed */
5939 static int
5940 xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
5941 {
5942         struct sockaddr_storage addr;
5943         socklen_t addrlen = 0;
5944         struct auth_master* master = xfr->task_probe->lookup_target;
5945         struct query_info qinfo;
5946         uint16_t qflags = BIT_RD;
5947         uint8_t dname[LDNS_MAX_DOMAINLEN+1];
5948         struct edns_data edns;
5949         sldns_buffer* buf = env->scratch_buffer;
5950         if(!master) return 0;
5951         if(extstrtoaddr(master->host, &addr, &addrlen)) {
5952                 /* not needed, host is in IP addr format */
5953                 return 0;
5954         }
5955         if(master->allow_notify && !master->http &&
5956                 strchr(master->host, '/') != NULL &&
5957                 strchr(master->host, '/') == strrchr(master->host, '/')) {
5958                 return 0; /* is IP/prefix format, not something to look up */
5959         }
5960
5961         /* use mesh_new_callback to probe for non-addr hosts,
5962          * and then wait for them to be looked up (in cache, or query) */
5963         qinfo.qname_len = sizeof(dname);
5964         if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
5965                 != 0) {
5966                 log_err("cannot parse host name of master %s", master->host);
5967                 return 0;
5968         }
5969         qinfo.qname = dname;
5970         qinfo.qclass = xfr->dclass;
5971         qinfo.qtype = LDNS_RR_TYPE_A;
5972         if(xfr->task_probe->lookup_aaaa)
5973                 qinfo.qtype = LDNS_RR_TYPE_AAAA;
5974         qinfo.local_alias = NULL;
5975         if(verbosity >= VERB_ALGO) {
5976                 char buf[512];
5977                 char buf2[LDNS_MAX_DOMAINLEN+1];
5978                 dname_str(xfr->name, buf2);
5979                 snprintf(buf, sizeof(buf), "auth zone %s: master lookup"
5980                         " for task_probe", buf2);
5981                 log_query_info(VERB_ALGO, buf, &qinfo);
5982         }
5983         edns.edns_present = 1;
5984         edns.ext_rcode = 0;
5985         edns.edns_version = 0;
5986         edns.bits = EDNS_DO;
5987         edns.opt_list = NULL;
5988         if(sldns_buffer_capacity(buf) < 65535)
5989                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
5990         else    edns.udp_size = 65535;
5991
5992         /* unlock xfr during mesh_new_callback() because the callback can be
5993          * called straight away */
5994         lock_basic_unlock(&xfr->lock);
5995         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
5996                 &auth_xfer_probe_lookup_callback, xfr)) {
5997                 lock_basic_lock(&xfr->lock);
5998                 log_err("out of memory lookup up master %s", master->host);
5999                 return 0;
6000         }
6001         lock_basic_lock(&xfr->lock);
6002         return 1;
6003 }
6004
6005 /** move to sending the probe packets, next if fails. task_probe */
6006 static void
6007 xfr_probe_send_or_end(struct auth_xfer* xfr, struct module_env* env)
6008 {
6009         /* are we doing hostname lookups? */
6010         while(xfr->task_probe->lookup_target) {
6011                 if(xfr_probe_lookup_host(xfr, env)) {
6012                         /* wait for lookup to finish,
6013                          * note that the hostname may be in unbound's cache
6014                          * and we may then get an instant cache response,
6015                          * and that calls the callback just like a full
6016                          * lookup and lookup failures also call callback */
6017                         lock_basic_unlock(&xfr->lock);
6018                         return;
6019                 }
6020                 xfr_probe_move_to_next_lookup(xfr, env);
6021         }
6022         /* probe of list has ended.  Create or refresh the list of of
6023          * allow_notify addrs */
6024         probe_copy_masters_for_allow_notify(xfr);
6025         if(xfr->task_probe->only_lookup) {
6026                 /* only wanted lookups for copy, stop probe and start wait */
6027                 xfr->task_probe->only_lookup = 0;
6028                 xfr_probe_disown(xfr);
6029                 if(xfr->task_nextprobe->worker == NULL)
6030                         xfr_set_timeout(xfr, env, 0, 0);
6031                 lock_basic_unlock(&xfr->lock);
6032                 return;
6033         }
6034
6035         /* send probe packets */
6036         while(!xfr_probe_end_of_list(xfr)) {
6037                 if(xfr_probe_send_probe(xfr, env, AUTH_PROBE_TIMEOUT)) {
6038                         /* successfully sent probe, wait for callback */
6039                         lock_basic_unlock(&xfr->lock);
6040                         return;
6041                 }
6042                 /* failed to send probe, next master */
6043                 xfr_probe_nextmaster(xfr);
6044         }
6045
6046         /* done with probe sequence, wait */
6047         if(xfr->task_probe->have_new_lease) {
6048                 /* if zone not updated, start the wait timer again */
6049                 verbose(VERB_ALGO, "auth_zone unchanged, new lease, wait");
6050                 xfr_probe_disown(xfr);
6051                 if(xfr->have_zone)
6052                         xfr->lease_time = *env->now;
6053                 if(xfr->task_nextprobe->worker == NULL)
6054                         xfr_set_timeout(xfr, env, 0, 0);
6055         } else {
6056                 /* we failed to send this as well, move to the wait task,
6057                  * use the shorter retry timeout */
6058                 xfr_probe_disown(xfr);
6059                 /* pick up the nextprobe task and wait */
6060                 if(xfr->task_nextprobe->worker == NULL)
6061                         xfr_set_timeout(xfr, env, 1, 0);
6062         }
6063
6064         lock_basic_unlock(&xfr->lock);
6065 }
6066
6067 /** callback for task_probe lookup of host name, of A or AAAA */
6068 void auth_xfer_probe_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
6069         enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
6070         int ATTR_UNUSED(was_ratelimited))
6071 {
6072         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6073         struct module_env* env;
6074         log_assert(xfr->task_probe);
6075         lock_basic_lock(&xfr->lock);
6076         env = xfr->task_probe->env;
6077         if(env->outnet->want_to_quit) {
6078                 lock_basic_unlock(&xfr->lock);
6079                 return; /* stop on quit */
6080         }
6081
6082         /* process result */
6083         if(rcode == LDNS_RCODE_NOERROR) {
6084                 uint16_t wanted_qtype = LDNS_RR_TYPE_A;
6085                 struct regional* temp = env->scratch;
6086                 struct query_info rq;
6087                 struct reply_info* rep;
6088                 if(xfr->task_probe->lookup_aaaa)
6089                         wanted_qtype = LDNS_RR_TYPE_AAAA;
6090                 memset(&rq, 0, sizeof(rq));
6091                 rep = parse_reply_in_temp_region(buf, temp, &rq);
6092                 if(rep && rq.qtype == wanted_qtype &&
6093                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
6094                         /* parsed successfully */
6095                         struct ub_packed_rrset_key* answer =
6096                                 reply_find_answer_rrset(&rq, rep);
6097                         if(answer) {
6098                                 xfr_master_add_addrs(xfr->task_probe->
6099                                         lookup_target, answer, wanted_qtype);
6100                         }
6101                 }
6102         }
6103         if(xfr->task_probe->lookup_target->list &&
6104                 xfr->task_probe->lookup_target == xfr_probe_current_master(xfr))
6105                 xfr->task_probe->scan_addr = xfr->task_probe->lookup_target->list;
6106
6107         /* move to lookup AAAA after A lookup, move to next hostname lookup,
6108          * or move to send the probes, or, if nothing to do, end task_probe */
6109         xfr_probe_move_to_next_lookup(xfr, env);
6110         xfr_probe_send_or_end(xfr, env);
6111 }
6112
6113 /** disown task_nextprobe.  caller must hold xfr.lock */
6114 static void
6115 xfr_nextprobe_disown(struct auth_xfer* xfr)
6116 {
6117         /* delete the timer, because the next worker to pick this up may
6118          * not have the same event base */
6119         comm_timer_delete(xfr->task_nextprobe->timer);
6120         xfr->task_nextprobe->timer = NULL;
6121         xfr->task_nextprobe->next_probe = 0;
6122         /* we don't own this item anymore */
6123         xfr->task_nextprobe->worker = NULL;
6124         xfr->task_nextprobe->env = NULL;
6125 }
6126
6127 /** xfer nextprobe timeout callback, this is part of task_nextprobe */
6128 void
6129 auth_xfer_timer(void* arg)
6130 {
6131         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6132         struct module_env* env;
6133         log_assert(xfr->task_nextprobe);
6134         lock_basic_lock(&xfr->lock);
6135         env = xfr->task_nextprobe->env;
6136         if(env->outnet->want_to_quit) {
6137                 lock_basic_unlock(&xfr->lock);
6138                 return; /* stop on quit */
6139         }
6140
6141         /* see if zone has expired, and if so, also set auth_zone expired */
6142         if(xfr->have_zone && !xfr->zone_expired &&
6143            *env->now >= xfr->lease_time + xfr->expiry) {
6144                 lock_basic_unlock(&xfr->lock);
6145                 auth_xfer_set_expired(xfr, env, 1);
6146                 lock_basic_lock(&xfr->lock);
6147         }
6148
6149         xfr_nextprobe_disown(xfr);
6150
6151         if(!xfr_start_probe(xfr, env, NULL)) {
6152                 /* not started because already in progress */
6153                 lock_basic_unlock(&xfr->lock);
6154         }
6155 }
6156
6157 /** return true if there are probe (SOA UDP query) targets in the master list*/
6158 static int
6159 have_probe_targets(struct auth_master* list)
6160 {
6161         struct auth_master* p;
6162         for(p=list; p; p = p->next) {
6163                 if(!p->allow_notify && p->host)
6164                         return 1;
6165         }
6166         return 0;
6167 }
6168
6169 /** start task_probe if possible, if no masters for probe start task_transfer
6170  * returns true if task has been started, and false if the task is already
6171  * in progress. */
6172 static int
6173 xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
6174         struct auth_master* spec)
6175 {
6176         /* see if we need to start a probe (or maybe it is already in
6177          * progress (due to notify)) */
6178         if(xfr->task_probe->worker == NULL) {
6179                 if(!have_probe_targets(xfr->task_probe->masters) &&
6180                         !(xfr->task_probe->only_lookup &&
6181                         xfr->task_probe->masters != NULL)) {
6182                         /* useless to pick up task_probe, no masters to
6183                          * probe. Instead attempt to pick up task transfer */
6184                         if(xfr->task_transfer->worker == NULL) {
6185                                 xfr_start_transfer(xfr, env, spec);
6186                                 return 1;
6187                         }
6188                         /* task transfer already in progress */
6189                         return 0;
6190                 }
6191
6192                 /* pick up the probe task ourselves */
6193                 xfr->task_probe->worker = env->worker;
6194                 xfr->task_probe->env = env;
6195                 xfr->task_probe->cp = NULL;
6196
6197                 /* start the task */
6198                 /* have not seen a new lease yet, this scan */
6199                 xfr->task_probe->have_new_lease = 0;
6200                 /* if this was a timeout, no specific first master to scan */
6201                 /* otherwise, spec is nonNULL the notified master, scan
6202                  * first and also transfer first from it */
6203                 xfr_probe_start_list(xfr, spec);
6204                 /* setup to start the lookup of hostnames of masters afresh */
6205                 xfr_probe_start_lookups(xfr);
6206                 /* send the probe packet or next send, or end task */
6207                 xfr_probe_send_or_end(xfr, env);
6208                 return 1;
6209         }
6210         return 0;
6211 }
6212
6213 /** for task_nextprobe.
6214  * determine next timeout for auth_xfer. Also (re)sets timer.
6215  * @param xfr: task structure
6216  * @param env: module environment, with worker and time.
6217  * @param failure: set true if timer should be set for failure retry.
6218  * @param lookup_only: only perform lookups when timer done, 0 sec timeout
6219  */
6220 static void
6221 xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
6222         int failure, int lookup_only)
6223 {
6224         struct timeval tv;
6225         log_assert(xfr->task_nextprobe != NULL);
6226         log_assert(xfr->task_nextprobe->worker == NULL ||
6227                 xfr->task_nextprobe->worker == env->worker);
6228         /* normally, nextprobe = startoflease + refresh,
6229          * but if expiry is sooner, use that one.
6230          * after a failure, use the retry timer instead. */
6231         xfr->task_nextprobe->next_probe = *env->now;
6232         if(xfr->lease_time && !failure)
6233                 xfr->task_nextprobe->next_probe = xfr->lease_time;
6234
6235         if(!failure) {
6236                 xfr->task_nextprobe->backoff = 0;
6237         } else {
6238                 if(xfr->task_nextprobe->backoff == 0)
6239                                 xfr->task_nextprobe->backoff = 3;
6240                 else    xfr->task_nextprobe->backoff *= 2;
6241                 if(xfr->task_nextprobe->backoff > AUTH_TRANSFER_MAX_BACKOFF)
6242                         xfr->task_nextprobe->backoff =
6243                                 AUTH_TRANSFER_MAX_BACKOFF;
6244         }
6245
6246         if(xfr->have_zone) {
6247                 time_t wait = xfr->refresh;
6248                 if(failure) wait = xfr->retry;
6249                 if(xfr->expiry < wait)
6250                         xfr->task_nextprobe->next_probe += xfr->expiry;
6251                 else    xfr->task_nextprobe->next_probe += wait;
6252                 if(failure)
6253                         xfr->task_nextprobe->next_probe +=
6254                                 xfr->task_nextprobe->backoff;
6255                 /* put the timer exactly on expiry, if possible */
6256                 if(xfr->lease_time && xfr->lease_time+xfr->expiry <
6257                         xfr->task_nextprobe->next_probe &&
6258                         xfr->lease_time+xfr->expiry > *env->now)
6259                         xfr->task_nextprobe->next_probe =
6260                                 xfr->lease_time+xfr->expiry;
6261         } else {
6262                 xfr->task_nextprobe->next_probe +=
6263                         xfr->task_nextprobe->backoff;
6264         }
6265
6266         if(!xfr->task_nextprobe->timer) {
6267                 xfr->task_nextprobe->timer = comm_timer_create(
6268                         env->worker_base, auth_xfer_timer, xfr);
6269                 if(!xfr->task_nextprobe->timer) {
6270                         /* failed to malloc memory. likely zone transfer
6271                          * also fails for that. skip the timeout */
6272                         char zname[255+1];
6273                         dname_str(xfr->name, zname);
6274                         log_err("cannot allocate timer, no refresh for %s",
6275                                 zname);
6276                         return;
6277                 }
6278         }
6279         xfr->task_nextprobe->worker = env->worker;
6280         xfr->task_nextprobe->env = env;
6281         if(*(xfr->task_nextprobe->env->now) <= xfr->task_nextprobe->next_probe)
6282                 tv.tv_sec = xfr->task_nextprobe->next_probe -
6283                         *(xfr->task_nextprobe->env->now);
6284         else    tv.tv_sec = 0;
6285         if(tv.tv_sec != 0 && lookup_only && xfr->task_probe->masters) {
6286                 /* don't lookup_only, if lookup timeout is 0 anyway,
6287                  * or if we don't have masters to lookup */
6288                 tv.tv_sec = 0;
6289                 if(xfr->task_probe && xfr->task_probe->worker == NULL)
6290                         xfr->task_probe->only_lookup = 1;
6291         }
6292         if(verbosity >= VERB_ALGO) {
6293                 char zname[255+1];
6294                 dname_str(xfr->name, zname);
6295                 verbose(VERB_ALGO, "auth zone %s timeout in %d seconds",
6296                         zname, (int)tv.tv_sec);
6297         }
6298         tv.tv_usec = 0;
6299         comm_timer_set(xfr->task_nextprobe->timer, &tv);
6300 }
6301
6302 /** initial pick up of worker timeouts, ties events to worker event loop */
6303 void
6304 auth_xfer_pickup_initial(struct auth_zones* az, struct module_env* env)
6305 {
6306         struct auth_xfer* x;
6307         lock_rw_wrlock(&az->lock);
6308         RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6309                 lock_basic_lock(&x->lock);
6310                 /* set lease_time, because we now have timestamp in env,
6311                  * (not earlier during startup and apply_cfg), and this
6312                  * notes the start time when the data was acquired */
6313                 if(x->have_zone)
6314                         x->lease_time = *env->now;
6315                 if(x->task_nextprobe && x->task_nextprobe->worker == NULL) {
6316                         xfr_set_timeout(x, env, 0, 1);
6317                 }
6318                 lock_basic_unlock(&x->lock);
6319         }
6320         lock_rw_unlock(&az->lock);
6321 }
6322
6323 void auth_zones_cleanup(struct auth_zones* az)
6324 {
6325         struct auth_xfer* x;
6326         lock_rw_wrlock(&az->lock);
6327         RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6328                 lock_basic_lock(&x->lock);
6329                 if(x->task_nextprobe && x->task_nextprobe->worker != NULL) {
6330                         xfr_nextprobe_disown(x);
6331                 }
6332                 if(x->task_probe && x->task_probe->worker != NULL) {
6333                         xfr_probe_disown(x);
6334                 }
6335                 if(x->task_transfer && x->task_transfer->worker != NULL) {
6336                         auth_chunks_delete(x->task_transfer);
6337                         xfr_transfer_disown(x);
6338                 }
6339                 lock_basic_unlock(&x->lock);
6340         }
6341         lock_rw_unlock(&az->lock);
6342 }
6343
6344 /**
6345  * malloc the xfer and tasks
6346  * @param z: auth_zone with name of zone.
6347  */
6348 static struct auth_xfer*
6349 auth_xfer_new(struct auth_zone* z)
6350 {
6351         struct auth_xfer* xfr;
6352         xfr = (struct auth_xfer*)calloc(1, sizeof(*xfr));
6353         if(!xfr) return NULL;
6354         xfr->name = memdup(z->name, z->namelen);
6355         if(!xfr->name) {
6356                 free(xfr);
6357                 return NULL;
6358         }
6359         xfr->node.key = xfr;
6360         xfr->namelen = z->namelen;
6361         xfr->namelabs = z->namelabs;
6362         xfr->dclass = z->dclass;
6363
6364         xfr->task_nextprobe = (struct auth_nextprobe*)calloc(1,
6365                 sizeof(struct auth_nextprobe));
6366         if(!xfr->task_nextprobe) {
6367                 free(xfr->name);
6368                 free(xfr);
6369                 return NULL;
6370         }
6371         xfr->task_probe = (struct auth_probe*)calloc(1,
6372                 sizeof(struct auth_probe));
6373         if(!xfr->task_probe) {
6374                 free(xfr->task_nextprobe);
6375                 free(xfr->name);
6376                 free(xfr);
6377                 return NULL;
6378         }
6379         xfr->task_transfer = (struct auth_transfer*)calloc(1,
6380                 sizeof(struct auth_transfer));
6381         if(!xfr->task_transfer) {
6382                 free(xfr->task_probe);
6383                 free(xfr->task_nextprobe);
6384                 free(xfr->name);
6385                 free(xfr);
6386                 return NULL;
6387         }
6388
6389         lock_basic_init(&xfr->lock);
6390         lock_protect(&xfr->lock, &xfr->name, sizeof(xfr->name));
6391         lock_protect(&xfr->lock, &xfr->namelen, sizeof(xfr->namelen));
6392         lock_protect(&xfr->lock, xfr->name, xfr->namelen);
6393         lock_protect(&xfr->lock, &xfr->namelabs, sizeof(xfr->namelabs));
6394         lock_protect(&xfr->lock, &xfr->dclass, sizeof(xfr->dclass));
6395         lock_protect(&xfr->lock, &xfr->notify_received, sizeof(xfr->notify_received));
6396         lock_protect(&xfr->lock, &xfr->notify_serial, sizeof(xfr->notify_serial));
6397         lock_protect(&xfr->lock, &xfr->zone_expired, sizeof(xfr->zone_expired));
6398         lock_protect(&xfr->lock, &xfr->have_zone, sizeof(xfr->have_zone));
6399         lock_protect(&xfr->lock, &xfr->serial, sizeof(xfr->serial));
6400         lock_protect(&xfr->lock, &xfr->retry, sizeof(xfr->retry));
6401         lock_protect(&xfr->lock, &xfr->refresh, sizeof(xfr->refresh));
6402         lock_protect(&xfr->lock, &xfr->expiry, sizeof(xfr->expiry));
6403         lock_protect(&xfr->lock, &xfr->lease_time, sizeof(xfr->lease_time));
6404         lock_protect(&xfr->lock, &xfr->task_nextprobe->worker,
6405                 sizeof(xfr->task_nextprobe->worker));
6406         lock_protect(&xfr->lock, &xfr->task_probe->worker,
6407                 sizeof(xfr->task_probe->worker));
6408         lock_protect(&xfr->lock, &xfr->task_transfer->worker,
6409                 sizeof(xfr->task_transfer->worker));
6410         lock_basic_lock(&xfr->lock);
6411         return xfr;
6412 }
6413
6414 /** Create auth_xfer structure.
6415  * This populates the have_zone, soa values, and so on times.
6416  * and sets the timeout, if a zone transfer is needed a short timeout is set.
6417  * For that the auth_zone itself must exist (and read in zonefile)
6418  * returns false on alloc failure. */
6419 struct auth_xfer*
6420 auth_xfer_create(struct auth_zones* az, struct auth_zone* z)
6421 {
6422         struct auth_xfer* xfr;
6423
6424         /* malloc it */
6425         xfr = auth_xfer_new(z);
6426         if(!xfr) {
6427                 log_err("malloc failure");
6428                 return NULL;
6429         }
6430         /* insert in tree */
6431         (void)rbtree_insert(&az->xtree, &xfr->node);
6432         return xfr;
6433 }
6434
6435 /** create new auth_master structure */
6436 static struct auth_master*
6437 auth_master_new(struct auth_master*** list)
6438 {
6439         struct auth_master *m;
6440         m = (struct auth_master*)calloc(1, sizeof(*m));
6441         if(!m) {
6442                 log_err("malloc failure");
6443                 return NULL;
6444         }
6445         /* set first pointer to m, or next pointer of previous element to m */
6446         (**list) = m;
6447         /* store m's next pointer as future point to store at */
6448         (*list) = &(m->next);
6449         return m;
6450 }
6451
6452 /** dup_prefix : create string from initial part of other string, malloced */
6453 static char*
6454 dup_prefix(char* str, size_t num)
6455 {
6456         char* result;
6457         size_t len = strlen(str);
6458         if(len < num) num = len; /* not more than strlen */
6459         result = (char*)malloc(num+1);
6460         if(!result) {
6461                 log_err("malloc failure");
6462                 return result;
6463         }
6464         memmove(result, str, num);
6465         result[num] = 0;
6466         return result;
6467 }
6468
6469 /** dup string and print error on error */
6470 static char*
6471 dup_all(char* str)
6472 {
6473         char* result = strdup(str);
6474         if(!result) {
6475                 log_err("malloc failure");
6476                 return NULL;
6477         }
6478         return result;
6479 }
6480
6481 /** find first of two characters */
6482 static char*
6483 str_find_first_of_chars(char* s, char a, char b)
6484 {
6485         char* ra = strchr(s, a);
6486         char* rb = strchr(s, b);
6487         if(!ra) return rb;
6488         if(!rb) return ra;
6489         if(ra < rb) return ra;
6490         return rb;
6491 }
6492
6493 /** parse URL into host and file parts, false on malloc or parse error */
6494 static int
6495 parse_url(char* url, char** host, char** file, int* port, int* ssl)
6496 {
6497         char* p = url;
6498         /* parse http://www.example.com/file.htm
6499          * or http://127.0.0.1   (index.html)
6500          * or https://[::1@1234]/a/b/c/d */
6501         *ssl = 1;
6502         *port = AUTH_HTTPS_PORT;
6503
6504         /* parse http:// or https:// */
6505         if(strncmp(p, "http://", 7) == 0) {
6506                 p += 7;
6507                 *ssl = 0;
6508                 *port = AUTH_HTTP_PORT;
6509         } else if(strncmp(p, "https://", 8) == 0) {
6510                 p += 8;
6511         } else if(strstr(p, "://") && strchr(p, '/') > strstr(p, "://") &&
6512                 strchr(p, ':') >= strstr(p, "://")) {
6513                 char* uri = dup_prefix(p, (size_t)(strstr(p, "://")-p));
6514                 log_err("protocol %s:// not supported (for url %s)",
6515                         uri?uri:"", p);
6516                 free(uri);
6517                 return 0;
6518         }
6519
6520         /* parse hostname part */
6521         if(p[0] == '[') {
6522                 char* end = strchr(p, ']');
6523                 p++; /* skip over [ */
6524                 if(end) {
6525                         *host = dup_prefix(p, (size_t)(end-p));
6526                         if(!*host) return 0;
6527                         p = end+1; /* skip over ] */
6528                 } else {
6529                         *host = dup_all(p);
6530                         if(!*host) return 0;
6531                         p = end;
6532                 }
6533         } else {
6534                 char* end = str_find_first_of_chars(p, ':', '/');
6535                 if(end) {
6536                         *host = dup_prefix(p, (size_t)(end-p));
6537                         if(!*host) return 0;
6538                 } else {
6539                         *host = dup_all(p);
6540                         if(!*host) return 0;
6541                 }
6542                 p = end; /* at next : or / or NULL */
6543         }
6544
6545         /* parse port number */
6546         if(p && p[0] == ':') {
6547                 char* end = NULL;
6548                 *port = strtol(p+1, &end, 10);
6549                 p = end;
6550         }
6551
6552         /* parse filename part */
6553         while(p && *p == '/')
6554                 p++;
6555         if(!p || p[0] == 0)
6556                 *file = strdup("index.html");
6557         else    *file = strdup(p);
6558         if(!*file) {
6559                 log_err("malloc failure");
6560                 return 0;
6561         }
6562         return 1;
6563 }
6564
6565 int
6566 xfer_set_masters(struct auth_master** list, struct config_auth* c,
6567         int with_http)
6568 {
6569         struct auth_master* m;
6570         struct config_strlist* p;
6571         /* list points to the first, or next pointer for the new element */
6572         while(*list) {
6573                 list = &( (*list)->next );
6574         }
6575         if(with_http)
6576           for(p = c->urls; p; p = p->next) {
6577                 m = auth_master_new(&list);
6578                 m->http = 1;
6579                 if(!parse_url(p->str, &m->host, &m->file, &m->port, &m->ssl))
6580                         return 0;
6581         }
6582         for(p = c->masters; p; p = p->next) {
6583                 m = auth_master_new(&list);
6584                 m->ixfr = 1; /* this flag is not configurable */
6585                 m->host = strdup(p->str);
6586                 if(!m->host) {
6587                         log_err("malloc failure");
6588                         return 0;
6589                 }
6590         }
6591         for(p = c->allow_notify; p; p = p->next) {
6592                 m = auth_master_new(&list);
6593                 m->allow_notify = 1;
6594                 m->host = strdup(p->str);
6595                 if(!m->host) {
6596                         log_err("malloc failure");
6597                         return 0;
6598                 }
6599         }
6600         return 1;
6601 }
6602
6603 #define SERIAL_BITS     32
6604 int
6605 compare_serial(uint32_t a, uint32_t b)
6606 {
6607         const uint32_t cutoff = ((uint32_t) 1 << (SERIAL_BITS - 1));
6608
6609         if (a == b) {
6610                 return 0;
6611         } else if ((a < b && b - a < cutoff) || (a > b && a - b > cutoff)) {
6612                 return -1;
6613         } else {
6614                 return 1;
6615         }
6616 }