contrib/unbound/services/authzone.c

   1 /*
   2  * services/authzone.c - authoritative zone that is locally hosted.
   3  *
   4  * Copyright (c) 2017, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains the functions for an authority zone.  This zone
  40  * is queried by the iterator, just like a stub or forward zone, but then
  41  * the data is locally held.
  42  */
  43
  44 #include "config.h"
  45 #include "services/authzone.h"
  46 #include "util/data/dname.h"
  47 #include "util/data/msgparse.h"
  48 #include "util/data/msgreply.h"
  49 #include "util/data/msgencode.h"
  50 #include "util/data/packed_rrset.h"
  51 #include "util/regional.h"
  52 #include "util/net_help.h"
  53 #include "util/netevent.h"
  54 #include "util/config_file.h"
  55 #include "util/log.h"
  56 #include "util/module.h"
  57 #include "util/random.h"
  58 #include "services/cache/dns.h"
  59 #include "services/outside_network.h"
  60 #include "services/listen_dnsport.h"
  61 #include "services/mesh.h"
  62 #include "sldns/rrdef.h"
  63 #include "sldns/pkthdr.h"
  64 #include "sldns/sbuffer.h"
  65 #include "sldns/str2wire.h"
  66 #include "sldns/wire2str.h"
  67 #include "sldns/parseutil.h"
  68 #include "sldns/keyraw.h"
  69 #include "validator/val_nsec3.h"
  70 #include "validator/val_nsec.h"
  71 #include "validator/val_secalgo.h"
  72 #include "validator/val_sigcrypt.h"
  73 #include "validator/val_anchor.h"
  74 #include "validator/val_utils.h"
  75 #include <ctype.h>
  76
  77 /** bytes to use for NSEC3 hash buffer. 20 for sha1 */
  78 #define N3HASHBUFLEN 32
  79 /** max number of CNAMEs we are willing to follow (in one answer) */
  80 #define MAX_CNAME_CHAIN 8
  81 /** timeout for probe packets for SOA */
  82 #define AUTH_PROBE_TIMEOUT 100 /* msec */
  83 /** when to stop with SOA probes (when exponential timeouts exceed this) */
  84 #define AUTH_PROBE_TIMEOUT_STOP 1000 /* msec */
  85 /* auth transfer timeout for TCP connections, in msec */
  86 #define AUTH_TRANSFER_TIMEOUT 10000 /* msec */
  87 /* auth transfer max backoff for failed tranfers and probes */
  88 #define AUTH_TRANSFER_MAX_BACKOFF 86400 /* sec */
  89 /* auth http port number */
  90 #define AUTH_HTTP_PORT 80
  91 /* auth https port number */
  92 #define AUTH_HTTPS_PORT 443
  93 /* max depth for nested $INCLUDEs */
  94 #define MAX_INCLUDE_DEPTH 10
  95 /** number of timeouts before we fallback from IXFR to AXFR,
  96  * because some versions of servers (eg. dnsmasq) drop IXFR packets. */
  97 #define NUM_TIMEOUTS_FALLBACK_IXFR 3
  98
  99 /** pick up nextprobe task to start waiting to perform transfer actions */
 100 static void xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
 101         int failure, int lookup_only);
 102 /** move to sending the probe packets, next if fails. task_probe */
 103 static void xfr_probe_send_or_end(struct auth_xfer* xfr,
 104         struct module_env* env);
 105 /** pick up probe task with specified(or NULL) destination first,
 106  * or transfer task if nothing to probe, or false if already in progress */
 107 static int xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
 108         struct auth_master* spec);
 109 /** delete xfer structure (not its tree entry) */
 110 static void auth_xfer_delete(struct auth_xfer* xfr);
 111
 112 /** create new dns_msg */
 113 static struct dns_msg*
 114 msg_create(struct regional* region, struct query_info* qinfo)
 115 {
 116         struct dns_msg* msg = (struct dns_msg*)regional_alloc(region,
 117                 sizeof(struct dns_msg));
 118         if(!msg)
 119                 return NULL;
 120         msg->qinfo.qname = regional_alloc_init(region, qinfo->qname,
 121                 qinfo->qname_len);
 122         if(!msg->qinfo.qname)
 123                 return NULL;
 124         msg->qinfo.qname_len = qinfo->qname_len;
 125         msg->qinfo.qtype = qinfo->qtype;
 126         msg->qinfo.qclass = qinfo->qclass;
 127         msg->qinfo.local_alias = NULL;
 128         /* non-packed reply_info, because it needs to grow the array */
 129         msg->rep = (struct reply_info*)regional_alloc_zero(region,
 130                 sizeof(struct reply_info)-sizeof(struct rrset_ref));
 131         if(!msg->rep)
 132                 return NULL;
 133         msg->rep->flags = (uint16_t)(BIT_QR | BIT_AA);
 134         msg->rep->authoritative = 1;
 135         msg->rep->qdcount = 1;
 136         /* rrsets is NULL, no rrsets yet */
 137         return msg;
 138 }
 139
 140 /** grow rrset array by one in msg */
 141 static int
 142 msg_grow_array(struct regional* region, struct dns_msg* msg)
 143 {
 144         if(msg->rep->rrsets == NULL) {
 145                 msg->rep->rrsets = regional_alloc_zero(region,
 146                         sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
 147                 if(!msg->rep->rrsets)
 148                         return 0;
 149         } else {
 150                 struct ub_packed_rrset_key** rrsets_old = msg->rep->rrsets;
 151                 msg->rep->rrsets = regional_alloc_zero(region,
 152                         sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
 153                 if(!msg->rep->rrsets)
 154                         return 0;
 155                 memmove(msg->rep->rrsets, rrsets_old,
 156                         sizeof(struct ub_packed_rrset_key*)*msg->rep->rrset_count);
 157         }
 158         return 1;
 159 }
 160
 161 /** get ttl of rrset */
 162 static time_t
 163 get_rrset_ttl(struct ub_packed_rrset_key* k)
 164 {
 165         struct packed_rrset_data* d = (struct packed_rrset_data*)
 166                 k->entry.data;
 167         return d->ttl;
 168 }
 169
 170 /** Copy rrset into region from domain-datanode and packet rrset */
 171 static struct ub_packed_rrset_key*
 172 auth_packed_rrset_copy_region(struct auth_zone* z, struct auth_data* node,
 173         struct auth_rrset* rrset, struct regional* region, time_t adjust)
 174 {
 175         struct ub_packed_rrset_key key;
 176         memset(&key, 0, sizeof(key));
 177         key.entry.key = &key;
 178         key.entry.data = rrset->data;
 179         key.rk.dname = node->name;
 180         key.rk.dname_len = node->namelen;
 181         key.rk.type = htons(rrset->type);
 182         key.rk.rrset_class = htons(z->dclass);
 183         key.entry.hash = rrset_key_hash(&key.rk);
 184         return packed_rrset_copy_region(&key, region, adjust);
 185 }
 186
 187 /** fix up msg->rep TTL and prefetch ttl */
 188 static void
 189 msg_ttl(struct dns_msg* msg)
 190 {
 191         if(msg->rep->rrset_count == 0) return;
 192         if(msg->rep->rrset_count == 1) {
 193                 msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
 194                 msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 195                 msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
 196         } else if(get_rrset_ttl(msg->rep->rrsets[msg->rep->rrset_count-1]) <
 197                 msg->rep->ttl) {
 198                 msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[
 199                         msg->rep->rrset_count-1]);
 200                 msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 201                 msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
 202         }
 203 }
 204
 205 /** see if rrset is a duplicate in the answer message */
 206 static int
 207 msg_rrset_duplicate(struct dns_msg* msg, uint8_t* nm, size_t nmlen,
 208         uint16_t type, uint16_t dclass)
 209 {
 210         size_t i;
 211         for(i=0; i<msg->rep->rrset_count; i++) {
 212                 struct ub_packed_rrset_key* k = msg->rep->rrsets[i];
 213                 if(ntohs(k->rk.type) == type && k->rk.dname_len == nmlen &&
 214                         ntohs(k->rk.rrset_class) == dclass &&
 215                         query_dname_compare(k->rk.dname, nm) == 0)
 216                         return 1;
 217         }
 218         return 0;
 219 }
 220
 221 /** add rrset to answer section (no auth, add rrsets yet) */
 222 static int
 223 msg_add_rrset_an(struct auth_zone* z, struct regional* region,
 224         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 225 {
 226         log_assert(msg->rep->ns_numrrsets == 0);
 227         log_assert(msg->rep->ar_numrrsets == 0);
 228         if(!rrset || !node)
 229                 return 1;
 230         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 231                 z->dclass))
 232                 return 1;
 233         /* grow array */
 234         if(!msg_grow_array(region, msg))
 235                 return 0;
 236         /* copy it */
 237         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 238                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 239                 return 0;
 240         msg->rep->rrset_count++;
 241         msg->rep->an_numrrsets++;
 242         msg_ttl(msg);
 243         return 1;
 244 }
 245
 246 /** add rrset to authority section (no additonal section rrsets yet) */
 247 static int
 248 msg_add_rrset_ns(struct auth_zone* z, struct regional* region,
 249         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 250 {
 251         log_assert(msg->rep->ar_numrrsets == 0);
 252         if(!rrset || !node)
 253                 return 1;
 254         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 255                 z->dclass))
 256                 return 1;
 257         /* grow array */
 258         if(!msg_grow_array(region, msg))
 259                 return 0;
 260         /* copy it */
 261         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 262                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 263                 return 0;
 264         msg->rep->rrset_count++;
 265         msg->rep->ns_numrrsets++;
 266         msg_ttl(msg);
 267         return 1;
 268 }
 269
 270 /** add rrset to additional section */
 271 static int
 272 msg_add_rrset_ar(struct auth_zone* z, struct regional* region,
 273         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 274 {
 275         if(!rrset || !node)
 276                 return 1;
 277         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 278                 z->dclass))
 279                 return 1;
 280         /* grow array */
 281         if(!msg_grow_array(region, msg))
 282                 return 0;
 283         /* copy it */
 284         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 285                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 286                 return 0;
 287         msg->rep->rrset_count++;
 288         msg->rep->ar_numrrsets++;
 289         msg_ttl(msg);
 290         return 1;
 291 }
 292
 293 struct auth_zones* auth_zones_create(void)
 294 {
 295         struct auth_zones* az = (struct auth_zones*)calloc(1, sizeof(*az));
 296         if(!az) {
 297                 log_err("out of memory");
 298                 return NULL;
 299         }
 300         rbtree_init(&az->ztree, &auth_zone_cmp);
 301         rbtree_init(&az->xtree, &auth_xfer_cmp);
 302         lock_rw_init(&az->lock);
 303         lock_protect(&az->lock, &az->ztree, sizeof(az->ztree));
 304         lock_protect(&az->lock, &az->xtree, sizeof(az->xtree));
 305         /* also lock protects the rbnode's in struct auth_zone, auth_xfer */
 306         lock_rw_init(&az->rpz_lock);
 307         lock_protect(&az->rpz_lock, &az->rpz_first, sizeof(az->rpz_first));
 308         return az;
 309 }
 310
 311 int auth_zone_cmp(const void* z1, const void* z2)
 312 {
 313         /* first sort on class, so that hierarchy can be maintained within
 314          * a class */
 315         struct auth_zone* a = (struct auth_zone*)z1;
 316         struct auth_zone* b = (struct auth_zone*)z2;
 317         int m;
 318         if(a->dclass != b->dclass) {
 319                 if(a->dclass < b->dclass)
 320                         return -1;
 321                 return 1;
 322         }
 323         /* sorted such that higher zones sort before lower zones (their
 324          * contents) */
 325         return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
 326 }
 327
 328 int auth_data_cmp(const void* z1, const void* z2)
 329 {
 330         struct auth_data* a = (struct auth_data*)z1;
 331         struct auth_data* b = (struct auth_data*)z2;
 332         int m;
 333         /* canonical sort, because DNSSEC needs that */
 334         return dname_canon_lab_cmp(a->name, a->namelabs, b->name,
 335                 b->namelabs, &m);
 336 }
 337
 338 int auth_xfer_cmp(const void* z1, const void* z2)
 339 {
 340         /* first sort on class, so that hierarchy can be maintained within
 341          * a class */
 342         struct auth_xfer* a = (struct auth_xfer*)z1;
 343         struct auth_xfer* b = (struct auth_xfer*)z2;
 344         int m;
 345         if(a->dclass != b->dclass) {
 346                 if(a->dclass < b->dclass)
 347                         return -1;
 348                 return 1;
 349         }
 350         /* sorted such that higher zones sort before lower zones (their
 351          * contents) */
 352         return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
 353 }
 354
 355 /** delete auth rrset node */
 356 static void
 357 auth_rrset_delete(struct auth_rrset* rrset)
 358 {
 359         if(!rrset) return;
 360         free(rrset->data);
 361         free(rrset);
 362 }
 363
 364 /** delete auth data domain node */
 365 static void
 366 auth_data_delete(struct auth_data* n)
 367 {
 368         struct auth_rrset* p, *np;
 369         if(!n) return;
 370         p = n->rrsets;
 371         while(p) {
 372                 np = p->next;
 373                 auth_rrset_delete(p);
 374                 p = np;
 375         }
 376         free(n->name);
 377         free(n);
 378 }
 379
 380 /** helper traverse to delete zones */
 381 static void
 382 auth_data_del(rbnode_type* n, void* ATTR_UNUSED(arg))
 383 {
 384         struct auth_data* z = (struct auth_data*)n->key;
 385         auth_data_delete(z);
 386 }
 387
 388 /** delete an auth zone structure (tree remove must be done elsewhere) */
 389 static void
 390 auth_zone_delete(struct auth_zone* z, struct auth_zones* az)
 391 {
 392         if(!z) return;
 393         lock_rw_destroy(&z->lock);
 394         traverse_postorder(&z->data, auth_data_del, NULL);
 395
 396         if(az && z->rpz) {
 397                 /* keep RPZ linked list intact */
 398                 lock_rw_wrlock(&az->rpz_lock);
 399                 if(z->rpz_az_prev)
 400                         z->rpz_az_prev->rpz_az_next = z->rpz_az_next;
 401                 else
 402                         az->rpz_first = z->rpz_az_next;
 403                 if(z->rpz_az_next)
 404                         z->rpz_az_next->rpz_az_prev = z->rpz_az_prev;
 405                 lock_rw_unlock(&az->rpz_lock);
 406         }
 407         if(z->rpz)
 408                 rpz_delete(z->rpz);
 409         free(z->name);
 410         free(z->zonefile);
 411         free(z);
 412 }
 413
 414 struct auth_zone*
 415 auth_zone_create(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 416         uint16_t dclass)
 417 {
 418         struct auth_zone* z = (struct auth_zone*)calloc(1, sizeof(*z));
 419         if(!z) {
 420                 return NULL;
 421         }
 422         z->node.key = z;
 423         z->dclass = dclass;
 424         z->namelen = nmlen;
 425         z->namelabs = dname_count_labels(nm);
 426         z->name = memdup(nm, nmlen);
 427         if(!z->name) {
 428                 free(z);
 429                 return NULL;
 430         }
 431         rbtree_init(&z->data, &auth_data_cmp);
 432         lock_rw_init(&z->lock);
 433         lock_protect(&z->lock, &z->name, sizeof(*z)-sizeof(rbnode_type)-
 434                         sizeof(&z->rpz_az_next)-sizeof(&z->rpz_az_prev));
 435         lock_rw_wrlock(&z->lock);
 436         /* z lock protects all, except rbtree itself and the rpz linked list
 437          * pointers, which are protected using az->lock */
 438         if(!rbtree_insert(&az->ztree, &z->node)) {
 439                 lock_rw_unlock(&z->lock);
 440                 auth_zone_delete(z, NULL);
 441                 log_warn("duplicate auth zone");
 442                 return NULL;
 443         }
 444         return z;
 445 }
 446
 447 struct auth_zone*
 448 auth_zone_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 449         uint16_t dclass)
 450 {
 451         struct auth_zone key;
 452         key.node.key = &key;
 453         key.dclass = dclass;
 454         key.name = nm;
 455         key.namelen = nmlen;
 456         key.namelabs = dname_count_labels(nm);
 457         return (struct auth_zone*)rbtree_search(&az->ztree, &key);
 458 }
 459
 460 struct auth_xfer*
 461 auth_xfer_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 462         uint16_t dclass)
 463 {
 464         struct auth_xfer key;
 465         key.node.key = &key;
 466         key.dclass = dclass;
 467         key.name = nm;
 468         key.namelen = nmlen;
 469         key.namelabs = dname_count_labels(nm);
 470         return (struct auth_xfer*)rbtree_search(&az->xtree, &key);
 471 }
 472
 473 /** find an auth zone or sorted less-or-equal, return true if exact */
 474 static int
 475 auth_zone_find_less_equal(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 476         uint16_t dclass, struct auth_zone** z)
 477 {
 478         struct auth_zone key;
 479         key.node.key = &key;
 480         key.dclass = dclass;
 481         key.name = nm;
 482         key.namelen = nmlen;
 483         key.namelabs = dname_count_labels(nm);
 484         return rbtree_find_less_equal(&az->ztree, &key, (rbnode_type**)z);
 485 }
 486
 487
 488 /** find the auth zone that is above the given name */
 489 struct auth_zone*
 490 auth_zones_find_zone(struct auth_zones* az, uint8_t* name, size_t name_len,
 491         uint16_t dclass)
 492 {
 493         uint8_t* nm = name;
 494         size_t nmlen = name_len;
 495         struct auth_zone* z;
 496         if(auth_zone_find_less_equal(az, nm, nmlen, dclass, &z)) {
 497                 /* exact match */
 498                 return z;
 499         } else {
 500                 /* less-or-nothing */
 501                 if(!z) return NULL; /* nothing smaller, nothing above it */
 502                 /* we found smaller name; smaller may be above the name,
 503                  * but not below it. */
 504                 nm = dname_get_shared_topdomain(z->name, name);
 505                 dname_count_size_labels(nm, &nmlen);
 506                 z = NULL;
 507         }
 508
 509         /* search up */
 510         while(!z) {
 511                 z = auth_zone_find(az, nm, nmlen, dclass);
 512                 if(z) return z;
 513                 if(dname_is_root(nm)) break;
 514                 dname_remove_label(&nm, &nmlen);
 515         }
 516         return NULL;
 517 }
 518
 519 /** find or create zone with name str. caller must have lock on az.
 520  * returns a wrlocked zone */
 521 static struct auth_zone*
 522 auth_zones_find_or_add_zone(struct auth_zones* az, char* name)
 523 {
 524         uint8_t nm[LDNS_MAX_DOMAINLEN+1];
 525         size_t nmlen = sizeof(nm);
 526         struct auth_zone* z;
 527
 528         if(sldns_str2wire_dname_buf(name, nm, &nmlen) != 0) {
 529                 log_err("cannot parse auth zone name: %s", name);
 530                 return 0;
 531         }
 532         z = auth_zone_find(az, nm, nmlen, LDNS_RR_CLASS_IN);
 533         if(!z) {
 534                 /* not found, create the zone */
 535                 z = auth_zone_create(az, nm, nmlen, LDNS_RR_CLASS_IN);
 536         } else {
 537                 lock_rw_wrlock(&z->lock);
 538         }
 539         return z;
 540 }
 541
 542 /** find or create xfer zone with name str. caller must have lock on az.
 543  * returns a locked xfer */
 544 static struct auth_xfer*
 545 auth_zones_find_or_add_xfer(struct auth_zones* az, struct auth_zone* z)
 546 {
 547         struct auth_xfer* x;
 548         x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
 549         if(!x) {
 550                 /* not found, create the zone */
 551                 x = auth_xfer_create(az, z);
 552         } else {
 553                 lock_basic_lock(&x->lock);
 554         }
 555         return x;
 556 }
 557
 558 int
 559 auth_zone_set_zonefile(struct auth_zone* z, char* zonefile)
 560 {
 561         if(z->zonefile) free(z->zonefile);
 562         if(zonefile == NULL) {
 563                 z->zonefile = NULL;
 564         } else {
 565                 z->zonefile = strdup(zonefile);
 566                 if(!z->zonefile) {
 567                         log_err("malloc failure");
 568                         return 0;
 569                 }
 570         }
 571         return 1;
 572 }
 573
 574 /** set auth zone fallback. caller must have lock on zone */
 575 int
 576 auth_zone_set_fallback(struct auth_zone* z, char* fallbackstr)
 577 {
 578         if(strcmp(fallbackstr, "yes") != 0 && strcmp(fallbackstr, "no") != 0){
 579                 log_err("auth zone fallback, expected yes or no, got %s",
 580                         fallbackstr);
 581                 return 0;
 582         }
 583         z->fallback_enabled = (strcmp(fallbackstr, "yes")==0);
 584         return 1;
 585 }
 586
 587 /** create domain with the given name */
 588 static struct auth_data*
 589 az_domain_create(struct auth_zone* z, uint8_t* nm, size_t nmlen)
 590 {
 591         struct auth_data* n = (struct auth_data*)malloc(sizeof(*n));
 592         if(!n) return NULL;
 593         memset(n, 0, sizeof(*n));
 594         n->node.key = n;
 595         n->name = memdup(nm, nmlen);
 596         if(!n->name) {
 597                 free(n);
 598                 return NULL;
 599         }
 600         n->namelen = nmlen;
 601         n->namelabs = dname_count_labels(nm);
 602         if(!rbtree_insert(&z->data, &n->node)) {
 603                 log_warn("duplicate auth domain name");
 604                 free(n->name);
 605                 free(n);
 606                 return NULL;
 607         }
 608         return n;
 609 }
 610
 611 /** find domain with exactly the given name */
 612 static struct auth_data*
 613 az_find_name(struct auth_zone* z, uint8_t* nm, size_t nmlen)
 614 {
 615         struct auth_zone key;
 616         key.node.key = &key;
 617         key.name = nm;
 618         key.namelen = nmlen;
 619         key.namelabs = dname_count_labels(nm);
 620         return (struct auth_data*)rbtree_search(&z->data, &key);
 621 }
 622
 623 /** Find domain name (or closest match) */
 624 static void
 625 az_find_domain(struct auth_zone* z, struct query_info* qinfo, int* node_exact,
 626         struct auth_data** node)
 627 {
 628         struct auth_zone key;
 629         key.node.key = &key;
 630         key.name = qinfo->qname;
 631         key.namelen = qinfo->qname_len;
 632         key.namelabs = dname_count_labels(key.name);
 633         *node_exact = rbtree_find_less_equal(&z->data, &key,
 634                 (rbnode_type**)node);
 635 }
 636
 637 /** find or create domain with name in zone */
 638 static struct auth_data*
 639 az_domain_find_or_create(struct auth_zone* z, uint8_t* dname,
 640         size_t dname_len)
 641 {
 642         struct auth_data* n = az_find_name(z, dname, dname_len);
 643         if(!n) {
 644                 n = az_domain_create(z, dname, dname_len);
 645         }
 646         return n;
 647 }
 648
 649 /** find rrset of given type in the domain */
 650 static struct auth_rrset*
 651 az_domain_rrset(struct auth_data* n, uint16_t t)
 652 {
 653         struct auth_rrset* rrset;
 654         if(!n) return NULL;
 655         rrset = n->rrsets;
 656         while(rrset) {
 657                 if(rrset->type == t)
 658                         return rrset;
 659                 rrset = rrset->next;
 660         }
 661         return NULL;
 662 }
 663
 664 /** remove rrset of this type from domain */
 665 static void
 666 domain_remove_rrset(struct auth_data* node, uint16_t rr_type)
 667 {
 668         struct auth_rrset* rrset, *prev;
 669         if(!node) return;
 670         prev = NULL;
 671         rrset = node->rrsets;
 672         while(rrset) {
 673                 if(rrset->type == rr_type) {
 674                         /* found it, now delete it */
 675                         if(prev) prev->next = rrset->next;
 676                         else    node->rrsets = rrset->next;
 677                         auth_rrset_delete(rrset);
 678                         return;
 679                 }
 680                 prev = rrset;
 681                 rrset = rrset->next;
 682         }
 683 }
 684
 685 /** find an rrsig index in the rrset.  returns true if found */
 686 static int
 687 az_rrset_find_rrsig(struct packed_rrset_data* d, uint8_t* rdata, size_t len,
 688         size_t* index)
 689 {
 690         size_t i;
 691         for(i=d->count; i<d->count + d->rrsig_count; i++) {
 692                 if(d->rr_len[i] != len)
 693                         continue;
 694                 if(memcmp(d->rr_data[i], rdata, len) == 0) {
 695                         *index = i;
 696                         return 1;
 697                 }
 698         }
 699         return 0;
 700 }
 701
 702 /** see if rdata is duplicate */
 703 static int
 704 rdata_duplicate(struct packed_rrset_data* d, uint8_t* rdata, size_t len)
 705 {
 706         size_t i;
 707         for(i=0; i<d->count + d->rrsig_count; i++) {
 708                 if(d->rr_len[i] != len)
 709                         continue;
 710                 if(memcmp(d->rr_data[i], rdata, len) == 0)
 711                         return 1;
 712         }
 713         return 0;
 714 }
 715
 716 /** get rrsig type covered from rdata.
 717  * @param rdata: rdata in wireformat, starting with 16bit rdlength.
 718  * @param rdatalen: length of rdata buffer.
 719  * @return type covered (or 0).
 720  */
 721 static uint16_t
 722 rrsig_rdata_get_type_covered(uint8_t* rdata, size_t rdatalen)
 723 {
 724         if(rdatalen < 4)
 725                 return 0;
 726         return sldns_read_uint16(rdata+2);
 727 }
 728
 729 /** remove RR from existing RRset. Also sig, if it is a signature.
 730  * reallocates the packed rrset for a new one, false on alloc failure */
 731 static int
 732 rrset_remove_rr(struct auth_rrset* rrset, size_t index)
 733 {
 734         struct packed_rrset_data* d, *old = rrset->data;
 735         size_t i;
 736         if(index >= old->count + old->rrsig_count)
 737                 return 0; /* index out of bounds */
 738         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old) - (
 739                 sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t) +
 740                 old->rr_len[index]));
 741         if(!d) {
 742                 log_err("malloc failure");
 743                 return 0;
 744         }
 745         d->ttl = old->ttl;
 746         d->count = old->count;
 747         d->rrsig_count = old->rrsig_count;
 748         if(index < d->count) d->count--;
 749         else d->rrsig_count--;
 750         d->trust = old->trust;
 751         d->security = old->security;
 752
 753         /* set rr_len, needed for ptr_fixup */
 754         d->rr_len = (size_t*)((uint8_t*)d +
 755                 sizeof(struct packed_rrset_data));
 756         if(index > 0)
 757                 memmove(d->rr_len, old->rr_len, (index)*sizeof(size_t));
 758         if(index+1 < old->count+old->rrsig_count)
 759                 memmove(&d->rr_len[index], &old->rr_len[index+1],
 760                 (old->count+old->rrsig_count - (index+1))*sizeof(size_t));
 761         packed_rrset_ptr_fixup(d);
 762
 763         /* move over ttls */
 764         if(index > 0)
 765                 memmove(d->rr_ttl, old->rr_ttl, (index)*sizeof(time_t));
 766         if(index+1 < old->count+old->rrsig_count)
 767                 memmove(&d->rr_ttl[index], &old->rr_ttl[index+1],
 768                 (old->count+old->rrsig_count - (index+1))*sizeof(time_t));
 769
 770         /* move over rr_data */
 771         for(i=0; i<d->count+d->rrsig_count; i++) {
 772                 size_t oldi;
 773                 if(i < index) oldi = i;
 774                 else oldi = i+1;
 775                 memmove(d->rr_data[i], old->rr_data[oldi], d->rr_len[i]);
 776         }
 777
 778         /* recalc ttl (lowest of remaining RR ttls) */
 779         if(d->count + d->rrsig_count > 0)
 780                 d->ttl = d->rr_ttl[0];
 781         for(i=0; i<d->count+d->rrsig_count; i++) {
 782                 if(d->rr_ttl[i] < d->ttl)
 783                         d->ttl = d->rr_ttl[i];
 784         }
 785
 786         free(rrset->data);
 787         rrset->data = d;
 788         return 1;
 789 }
 790
 791 /** add RR to existing RRset. If insert_sig is true, add to rrsigs.
 792  * This reallocates the packed rrset for a new one */
 793 static int
 794 rrset_add_rr(struct auth_rrset* rrset, uint32_t rr_ttl, uint8_t* rdata,
 795         size_t rdatalen, int insert_sig)
 796 {
 797         struct packed_rrset_data* d, *old = rrset->data;
 798         size_t total, old_total;
 799
 800         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
 801                 + sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t)
 802                 + rdatalen);
 803         if(!d) {
 804                 log_err("out of memory");
 805                 return 0;
 806         }
 807         /* copy base values */
 808         memcpy(d, old, sizeof(struct packed_rrset_data));
 809         if(!insert_sig) {
 810                 d->count++;
 811         } else {
 812                 d->rrsig_count++;
 813         }
 814         old_total = old->count + old->rrsig_count;
 815         total = d->count + d->rrsig_count;
 816         /* set rr_len, needed for ptr_fixup */
 817         d->rr_len = (size_t*)((uint8_t*)d +
 818                 sizeof(struct packed_rrset_data));
 819         if(old->count != 0)
 820                 memmove(d->rr_len, old->rr_len, old->count*sizeof(size_t));
 821         if(old->rrsig_count != 0)
 822                 memmove(d->rr_len+d->count, old->rr_len+old->count,
 823                         old->rrsig_count*sizeof(size_t));
 824         if(!insert_sig)
 825                 d->rr_len[d->count-1] = rdatalen;
 826         else    d->rr_len[total-1] = rdatalen;
 827         packed_rrset_ptr_fixup(d);
 828         if((time_t)rr_ttl < d->ttl)
 829                 d->ttl = rr_ttl;
 830
 831         /* copy old values into new array */
 832         if(old->count != 0) {
 833                 memmove(d->rr_ttl, old->rr_ttl, old->count*sizeof(time_t));
 834                 /* all the old rr pieces are allocated sequential, so we
 835                  * can copy them in one go */
 836                 memmove(d->rr_data[0], old->rr_data[0],
 837                         (old->rr_data[old->count-1] - old->rr_data[0]) +
 838                         old->rr_len[old->count-1]);
 839         }
 840         if(old->rrsig_count != 0) {
 841                 memmove(d->rr_ttl+d->count, old->rr_ttl+old->count,
 842                         old->rrsig_count*sizeof(time_t));
 843                 memmove(d->rr_data[d->count], old->rr_data[old->count],
 844                         (old->rr_data[old_total-1] - old->rr_data[old->count]) +
 845                         old->rr_len[old_total-1]);
 846         }
 847
 848         /* insert new value */
 849         if(!insert_sig) {
 850                 d->rr_ttl[d->count-1] = rr_ttl;
 851                 memmove(d->rr_data[d->count-1], rdata, rdatalen);
 852         } else {
 853                 d->rr_ttl[total-1] = rr_ttl;
 854                 memmove(d->rr_data[total-1], rdata, rdatalen);
 855         }
 856
 857         rrset->data = d;
 858         free(old);
 859         return 1;
 860 }
 861
 862 /** Create new rrset for node with packed rrset with one RR element */
 863 static struct auth_rrset*
 864 rrset_create(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
 865         uint8_t* rdata, size_t rdatalen)
 866 {
 867         struct auth_rrset* rrset = (struct auth_rrset*)calloc(1,
 868                 sizeof(*rrset));
 869         struct auth_rrset* p, *prev;
 870         struct packed_rrset_data* d;
 871         if(!rrset) {
 872                 log_err("out of memory");
 873                 return NULL;
 874         }
 875         rrset->type = rr_type;
 876
 877         /* the rrset data structure, with one RR */
 878         d = (struct packed_rrset_data*)calloc(1,
 879                 sizeof(struct packed_rrset_data) + sizeof(size_t) +
 880                 sizeof(uint8_t*) + sizeof(time_t) + rdatalen);
 881         if(!d) {
 882                 free(rrset);
 883                 log_err("out of memory");
 884                 return NULL;
 885         }
 886         rrset->data = d;
 887         d->ttl = rr_ttl;
 888         d->trust = rrset_trust_prim_noglue;
 889         d->rr_len = (size_t*)((uint8_t*)d + sizeof(struct packed_rrset_data));
 890         d->rr_data = (uint8_t**)&(d->rr_len[1]);
 891         d->rr_ttl = (time_t*)&(d->rr_data[1]);
 892         d->rr_data[0] = (uint8_t*)&(d->rr_ttl[1]);
 893
 894         /* insert the RR */
 895         d->rr_len[0] = rdatalen;
 896         d->rr_ttl[0] = rr_ttl;
 897         memmove(d->rr_data[0], rdata, rdatalen);
 898         d->count++;
 899
 900         /* insert rrset into linked list for domain */
 901         /* find sorted place to link the rrset into the list */
 902         prev = NULL;
 903         p = node->rrsets;
 904         while(p && p->type<=rr_type) {
 905                 prev = p;
 906                 p = p->next;
 907         }
 908         /* so, prev is smaller, and p is larger than rr_type */
 909         rrset->next = p;
 910         if(prev) prev->next = rrset;
 911         else node->rrsets = rrset;
 912         return rrset;
 913 }
 914
 915 /** count number (and size) of rrsigs that cover a type */
 916 static size_t
 917 rrsig_num_that_cover(struct auth_rrset* rrsig, uint16_t rr_type, size_t* sigsz)
 918 {
 919         struct packed_rrset_data* d = rrsig->data;
 920         size_t i, num = 0;
 921         *sigsz = 0;
 922         log_assert(d && rrsig->type == LDNS_RR_TYPE_RRSIG);
 923         for(i=0; i<d->count+d->rrsig_count; i++) {
 924                 if(rrsig_rdata_get_type_covered(d->rr_data[i],
 925                         d->rr_len[i]) == rr_type) {
 926                         num++;
 927                         (*sigsz) += d->rr_len[i];
 928                 }
 929         }
 930         return num;
 931 }
 932
 933 /** See if rrsig set has covered sigs for rrset and move them over */
 934 static int
 935 rrset_moveover_rrsigs(struct auth_data* node, uint16_t rr_type,
 936         struct auth_rrset* rrset, struct auth_rrset* rrsig)
 937 {
 938         size_t sigs, sigsz, i, j, total;
 939         struct packed_rrset_data* sigold = rrsig->data;
 940         struct packed_rrset_data* old = rrset->data;
 941         struct packed_rrset_data* d, *sigd;
 942
 943         log_assert(rrset->type == rr_type);
 944         log_assert(rrsig->type == LDNS_RR_TYPE_RRSIG);
 945         sigs = rrsig_num_that_cover(rrsig, rr_type, &sigsz);
 946         if(sigs == 0) {
 947                 /* 0 rrsigs to move over, done */
 948                 return 1;
 949         }
 950
 951         /* allocate rrset sigsz larger for extra sigs elements, and
 952          * allocate rrsig sigsz smaller for less sigs elements. */
 953         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
 954                 + sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
 955                 + sigsz);
 956         if(!d) {
 957                 log_err("out of memory");
 958                 return 0;
 959         }
 960         /* copy base values */
 961         total = old->count + old->rrsig_count;
 962         memcpy(d, old, sizeof(struct packed_rrset_data));
 963         d->rrsig_count += sigs;
 964         /* setup rr_len */
 965         d->rr_len = (size_t*)((uint8_t*)d +
 966                 sizeof(struct packed_rrset_data));
 967         if(total != 0)
 968                 memmove(d->rr_len, old->rr_len, total*sizeof(size_t));
 969         j = d->count+d->rrsig_count-sigs;
 970         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
 971                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
 972                         sigold->rr_len[i]) == rr_type) {
 973                         d->rr_len[j] = sigold->rr_len[i];
 974                         j++;
 975                 }
 976         }
 977         packed_rrset_ptr_fixup(d);
 978
 979         /* copy old values into new array */
 980         if(total != 0) {
 981                 memmove(d->rr_ttl, old->rr_ttl, total*sizeof(time_t));
 982                 /* all the old rr pieces are allocated sequential, so we
 983                  * can copy them in one go */
 984                 memmove(d->rr_data[0], old->rr_data[0],
 985                         (old->rr_data[total-1] - old->rr_data[0]) +
 986                         old->rr_len[total-1]);
 987         }
 988
 989         /* move over the rrsigs to the larger rrset*/
 990         j = d->count+d->rrsig_count-sigs;
 991         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
 992                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
 993                         sigold->rr_len[i]) == rr_type) {
 994                         /* move this one over to location j */
 995                         d->rr_ttl[j] = sigold->rr_ttl[i];
 996                         memmove(d->rr_data[j], sigold->rr_data[i],
 997                                 sigold->rr_len[i]);
 998                         if(d->rr_ttl[j] < d->ttl)
 999                                 d->ttl = d->rr_ttl[j];
1000                         j++;
1001                 }
1002         }
1003
1004         /* put it in and deallocate the old rrset */
1005         rrset->data = d;
1006         free(old);
1007
1008         /* now make rrsig set smaller */
1009         if(sigold->count+sigold->rrsig_count == sigs) {
1010                 /* remove all sigs from rrsig, remove it entirely */
1011                 domain_remove_rrset(node, LDNS_RR_TYPE_RRSIG);
1012                 return 1;
1013         }
1014         log_assert(packed_rrset_sizeof(sigold) > sigs*(sizeof(size_t) +
1015                 sizeof(uint8_t*) + sizeof(time_t)) + sigsz);
1016         sigd = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(sigold)
1017                 - sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
1018                 - sigsz);
1019         if(!sigd) {
1020                 /* no need to free up d, it has already been placed in the
1021                  * node->rrset structure */
1022                 log_err("out of memory");
1023                 return 0;
1024         }
1025         /* copy base values */
1026         memcpy(sigd, sigold, sizeof(struct packed_rrset_data));
1027         /* in sigd the RRSIGs are stored in the base of the RR, in count */
1028         sigd->count -= sigs;
1029         /* setup rr_len */
1030         sigd->rr_len = (size_t*)((uint8_t*)sigd +
1031                 sizeof(struct packed_rrset_data));
1032         j = 0;
1033         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1034                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1035                         sigold->rr_len[i]) != rr_type) {
1036                         sigd->rr_len[j] = sigold->rr_len[i];
1037                         j++;
1038                 }
1039         }
1040         packed_rrset_ptr_fixup(sigd);
1041
1042         /* copy old values into new rrsig array */
1043         j = 0;
1044         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1045                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1046                         sigold->rr_len[i]) != rr_type) {
1047                         /* move this one over to location j */
1048                         sigd->rr_ttl[j] = sigold->rr_ttl[i];
1049                         memmove(sigd->rr_data[j], sigold->rr_data[i],
1050                                 sigold->rr_len[i]);
1051                         if(j==0) sigd->ttl = sigd->rr_ttl[j];
1052                         else {
1053                                 if(sigd->rr_ttl[j] < sigd->ttl)
1054                                         sigd->ttl = sigd->rr_ttl[j];
1055                         }
1056                         j++;
1057                 }
1058         }
1059
1060         /* put it in and deallocate the old rrset */
1061         rrsig->data = sigd;
1062         free(sigold);
1063
1064         return 1;
1065 }
1066
1067 /** copy the rrsigs from the rrset to the rrsig rrset, because the rrset
1068  * is going to be deleted.  reallocates the RRSIG rrset data. */
1069 static int
1070 rrsigs_copy_from_rrset_to_rrsigset(struct auth_rrset* rrset,
1071         struct auth_rrset* rrsigset)
1072 {
1073         size_t i;
1074         if(rrset->data->rrsig_count == 0)
1075                 return 1;
1076
1077         /* move them over one by one, because there might be duplicates,
1078          * duplicates are ignored */
1079         for(i=rrset->data->count;
1080                 i<rrset->data->count+rrset->data->rrsig_count; i++) {
1081                 uint8_t* rdata = rrset->data->rr_data[i];
1082                 size_t rdatalen = rrset->data->rr_len[i];
1083                 time_t rr_ttl  = rrset->data->rr_ttl[i];
1084
1085                 if(rdata_duplicate(rrsigset->data, rdata, rdatalen)) {
1086                         continue;
1087                 }
1088                 if(!rrset_add_rr(rrsigset, rr_ttl, rdata, rdatalen, 0))
1089                         return 0;
1090         }
1091         return 1;
1092 }
1093
1094 /** Add rr to node, ignores duplicate RRs,
1095  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1096 static int
1097 az_domain_add_rr(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
1098         uint8_t* rdata, size_t rdatalen, int* duplicate)
1099 {
1100         struct auth_rrset* rrset;
1101         /* packed rrsets have their rrsigs along with them, sort them out */
1102         if(rr_type == LDNS_RR_TYPE_RRSIG) {
1103                 uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1104                 if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1105                         /* a node of the correct type exists, add the RRSIG
1106                          * to the rrset of the covered data type */
1107                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1108                                 if(duplicate) *duplicate = 1;
1109                                 return 1;
1110                         }
1111                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 1))
1112                                 return 0;
1113                 } else if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1114                         /* add RRSIG to rrset of type RRSIG */
1115                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1116                                 if(duplicate) *duplicate = 1;
1117                                 return 1;
1118                         }
1119                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1120                                 return 0;
1121                 } else {
1122                         /* create rrset of type RRSIG */
1123                         if(!rrset_create(node, rr_type, rr_ttl, rdata,
1124                                 rdatalen))
1125                                 return 0;
1126                 }
1127         } else {
1128                 /* normal RR type */
1129                 if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1130                         /* add data to existing node with data type */
1131                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1132                                 if(duplicate) *duplicate = 1;
1133                                 return 1;
1134                         }
1135                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1136                                 return 0;
1137                 } else {
1138                         struct auth_rrset* rrsig;
1139                         /* create new node with data type */
1140                         if(!(rrset=rrset_create(node, rr_type, rr_ttl, rdata,
1141                                 rdatalen)))
1142                                 return 0;
1143
1144                         /* see if node of type RRSIG has signatures that
1145                          * cover the data type, and move them over */
1146                         /* and then make the RRSIG type smaller */
1147                         if((rrsig=az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
1148                                 != NULL) {
1149                                 if(!rrset_moveover_rrsigs(node, rr_type,
1150                                         rrset, rrsig))
1151                                         return 0;
1152                         }
1153                 }
1154         }
1155         return 1;
1156 }
1157
1158 /** insert RR into zone, ignore duplicates */
1159 static int
1160 az_insert_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1161         size_t dname_len, int* duplicate)
1162 {
1163         struct auth_data* node;
1164         uint8_t* dname = rr;
1165         uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1166         uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1167         uint32_t rr_ttl = sldns_wirerr_get_ttl(rr, rr_len, dname_len);
1168         size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1169                 dname_len))+2;
1170         /* rdata points to rdata prefixed with uint16 rdatalength */
1171         uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1172
1173         if(rr_class != z->dclass) {
1174                 log_err("wrong class for RR");
1175                 return 0;
1176         }
1177         if(!(node=az_domain_find_or_create(z, dname, dname_len))) {
1178                 log_err("cannot create domain");
1179                 return 0;
1180         }
1181         if(!az_domain_add_rr(node, rr_type, rr_ttl, rdata, rdatalen,
1182                 duplicate)) {
1183                 log_err("cannot add RR to domain");
1184                 return 0;
1185         }
1186         if(z->rpz) {
1187                 if(!(rpz_insert_rr(z->rpz, z->name, z->namelen, dname,
1188                         dname_len, rr_type, rr_class, rr_ttl, rdata, rdatalen,
1189                         rr, rr_len)))
1190                         return 0;
1191         }
1192         return 1;
1193 }
1194
1195 /** Remove rr from node, ignores nonexisting RRs,
1196  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1197 static int
1198 az_domain_remove_rr(struct auth_data* node, uint16_t rr_type,
1199         uint8_t* rdata, size_t rdatalen, int* nonexist)
1200 {
1201         struct auth_rrset* rrset;
1202         size_t index = 0;
1203
1204         /* find the plain RR of the given type */
1205         if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1206                 if(packed_rrset_find_rr(rrset->data, rdata, rdatalen, &index)) {
1207                         if(rrset->data->count == 1 &&
1208                                 rrset->data->rrsig_count == 0) {
1209                                 /* last RR, delete the rrset */
1210                                 domain_remove_rrset(node, rr_type);
1211                         } else if(rrset->data->count == 1 &&
1212                                 rrset->data->rrsig_count != 0) {
1213                                 /* move RRSIGs to the RRSIG rrset, or
1214                                  * this one becomes that RRset */
1215                                 struct auth_rrset* rrsigset = az_domain_rrset(
1216                                         node, LDNS_RR_TYPE_RRSIG);
1217                                 if(rrsigset) {
1218                                         /* move left over rrsigs to the
1219                                          * existing rrset of type RRSIG */
1220                                         rrsigs_copy_from_rrset_to_rrsigset(
1221                                                 rrset, rrsigset);
1222                                         /* and then delete the rrset */
1223                                         domain_remove_rrset(node, rr_type);
1224                                 } else {
1225                                         /* no rrset of type RRSIG, this
1226                                          * set is now of that type,
1227                                          * just remove the rr */
1228                                         if(!rrset_remove_rr(rrset, index))
1229                                                 return 0;
1230                                         rrset->type = LDNS_RR_TYPE_RRSIG;
1231                                         rrset->data->count = rrset->data->rrsig_count;
1232                                         rrset->data->rrsig_count = 0;
1233                                 }
1234                         } else {
1235                                 /* remove the RR from the rrset */
1236                                 if(!rrset_remove_rr(rrset, index))
1237                                         return 0;
1238                         }
1239                         return 1;
1240                 }
1241                 /* rr not found in rrset */
1242         }
1243
1244         /* is it a type RRSIG, look under the covered type */
1245         if(rr_type == LDNS_RR_TYPE_RRSIG) {
1246                 uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1247                 if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1248                         if(az_rrset_find_rrsig(rrset->data, rdata, rdatalen,
1249                                 &index)) {
1250                                 /* rrsig should have d->count > 0, be
1251                                  * over some rr of that type */
1252                                 /* remove the rrsig from the rrsigs list of the
1253                                  * rrset */
1254                                 if(!rrset_remove_rr(rrset, index))
1255                                         return 0;
1256                                 return 1;
1257                         }
1258                 }
1259                 /* also RRSIG not found */
1260         }
1261
1262         /* nothing found to delete */
1263         if(nonexist) *nonexist = 1;
1264         return 1;
1265 }
1266
1267 /** remove RR from zone, ignore if it does not exist, false on alloc failure*/
1268 static int
1269 az_remove_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1270         size_t dname_len, int* nonexist)
1271 {
1272         struct auth_data* node;
1273         uint8_t* dname = rr;
1274         uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1275         uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1276         size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1277                 dname_len))+2;
1278         /* rdata points to rdata prefixed with uint16 rdatalength */
1279         uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1280
1281         if(rr_class != z->dclass) {
1282                 log_err("wrong class for RR");
1283                 /* really also a nonexisting entry, because no records
1284                  * of that class in the zone, but return an error because
1285                  * getting records of the wrong class is a failure of the
1286                  * zone transfer */
1287                 return 0;
1288         }
1289         node = az_find_name(z, dname, dname_len);
1290         if(!node) {
1291                 /* node with that name does not exist */
1292                 /* nonexisting entry, because no such name */
1293                 *nonexist = 1;
1294                 return 1;
1295         }
1296         if(!az_domain_remove_rr(node, rr_type, rdata, rdatalen, nonexist)) {
1297                 /* alloc failure or so */
1298                 return 0;
1299         }
1300         /* remove the node, if necessary */
1301         /* an rrsets==NULL entry is not kept around for empty nonterminals,
1302          * and also parent nodes are not kept around, so we just delete it */
1303         if(node->rrsets == NULL) {
1304                 (void)rbtree_delete(&z->data, node);
1305                 auth_data_delete(node);
1306         }
1307         if(z->rpz) {
1308                 rpz_remove_rr(z->rpz, z->namelen, dname, dname_len, rr_type,
1309                         rr_class, rdata, rdatalen);
1310         }
1311         return 1;
1312 }
1313
1314 /** decompress an RR into the buffer where it'll be an uncompressed RR
1315  * with uncompressed dname and uncompressed rdata (dnames) */
1316 static int
1317 decompress_rr_into_buffer(struct sldns_buffer* buf, uint8_t* pkt,
1318         size_t pktlen, uint8_t* dname, uint16_t rr_type, uint16_t rr_class,
1319         uint32_t rr_ttl, uint8_t* rr_data, uint16_t rr_rdlen)
1320 {
1321         sldns_buffer pktbuf;
1322         size_t dname_len = 0;
1323         size_t rdlenpos;
1324         size_t rdlen;
1325         uint8_t* rd;
1326         const sldns_rr_descriptor* desc;
1327         sldns_buffer_init_frm_data(&pktbuf, pkt, pktlen);
1328         sldns_buffer_clear(buf);
1329
1330         /* decompress dname */
1331         sldns_buffer_set_position(&pktbuf,
1332                 (size_t)(dname - sldns_buffer_current(&pktbuf)));
1333         dname_len = pkt_dname_len(&pktbuf);
1334         if(dname_len == 0) return 0; /* parse fail on dname */
1335         if(!sldns_buffer_available(buf, dname_len)) return 0;
1336         dname_pkt_copy(&pktbuf, sldns_buffer_current(buf), dname);
1337         sldns_buffer_skip(buf, (ssize_t)dname_len);
1338
1339         /* type, class, ttl and rdatalength fields */
1340         if(!sldns_buffer_available(buf, 10)) return 0;
1341         sldns_buffer_write_u16(buf, rr_type);
1342         sldns_buffer_write_u16(buf, rr_class);
1343         sldns_buffer_write_u32(buf, rr_ttl);
1344         rdlenpos = sldns_buffer_position(buf);
1345         sldns_buffer_write_u16(buf, 0); /* rd length position */
1346
1347         /* decompress rdata */
1348         desc = sldns_rr_descript(rr_type);
1349         rd = rr_data;
1350         rdlen = rr_rdlen;
1351         if(rdlen > 0 && desc && desc->_dname_count > 0) {
1352                 int count = (int)desc->_dname_count;
1353                 int rdf = 0;
1354                 size_t len; /* how much rdata to plain copy */
1355                 size_t uncompressed_len, compressed_len;
1356                 size_t oldpos;
1357                 /* decompress dnames. */
1358                 while(rdlen > 0 && count) {
1359                         switch(desc->_wireformat[rdf]) {
1360                         case LDNS_RDF_TYPE_DNAME:
1361                                 sldns_buffer_set_position(&pktbuf,
1362                                         (size_t)(rd -
1363                                         sldns_buffer_begin(&pktbuf)));
1364                                 oldpos = sldns_buffer_position(&pktbuf);
1365                                 /* moves pktbuf to right after the
1366                                  * compressed dname, and returns uncompressed
1367                                  * dname length */
1368                                 uncompressed_len = pkt_dname_len(&pktbuf);
1369                                 if(!uncompressed_len)
1370                                         return 0; /* parse error in dname */
1371                                 if(!sldns_buffer_available(buf,
1372                                         uncompressed_len))
1373                                         /* dname too long for buffer */
1374                                         return 0;
1375                                 dname_pkt_copy(&pktbuf,
1376                                         sldns_buffer_current(buf), rd);
1377                                 sldns_buffer_skip(buf, (ssize_t)uncompressed_len);
1378                                 compressed_len = sldns_buffer_position(
1379                                         &pktbuf) - oldpos;
1380                                 rd += compressed_len;
1381                                 rdlen -= compressed_len;
1382                                 count--;
1383                                 len = 0;
1384                                 break;
1385                         case LDNS_RDF_TYPE_STR:
1386                                 len = rd[0] + 1;
1387                                 break;
1388                         default:
1389                                 len = get_rdf_size(desc->_wireformat[rdf]);
1390                                 break;
1391                         }
1392                         if(len) {
1393                                 if(!sldns_buffer_available(buf, len))
1394                                         return 0; /* too long for buffer */
1395                                 sldns_buffer_write(buf, rd, len);
1396                                 rd += len;
1397                                 rdlen -= len;
1398                         }
1399                         rdf++;
1400                 }
1401         }
1402         /* copy remaining data */
1403         if(rdlen > 0) {
1404                 if(!sldns_buffer_available(buf, rdlen)) return 0;
1405                 sldns_buffer_write(buf, rd, rdlen);
1406         }
1407         /* fixup rdlength */
1408         sldns_buffer_write_u16_at(buf, rdlenpos,
1409                 sldns_buffer_position(buf)-rdlenpos-2);
1410         sldns_buffer_flip(buf);
1411         return 1;
1412 }
1413
1414 /** insert RR into zone, from packet, decompress RR,
1415  * if duplicate is nonNULL set the flag but otherwise ignore duplicates */
1416 static int
1417 az_insert_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1418         struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1419         uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1420         uint16_t rr_rdlen, int* duplicate)
1421 {
1422         uint8_t* rr;
1423         size_t rr_len;
1424         size_t dname_len;
1425         if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1426                 rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1427                 log_err("could not decompress RR");
1428                 return 0;
1429         }
1430         rr = sldns_buffer_begin(scratch_buffer);
1431         rr_len = sldns_buffer_limit(scratch_buffer);
1432         dname_len = dname_valid(rr, rr_len);
1433         return az_insert_rr(z, rr, rr_len, dname_len, duplicate);
1434 }
1435
1436 /** remove RR from zone, from packet, decompress RR,
1437  * if nonexist is nonNULL set the flag but otherwise ignore nonexisting entries*/
1438 static int
1439 az_remove_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1440         struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1441         uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1442         uint16_t rr_rdlen, int* nonexist)
1443 {
1444         uint8_t* rr;
1445         size_t rr_len;
1446         size_t dname_len;
1447         if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1448                 rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1449                 log_err("could not decompress RR");
1450                 return 0;
1451         }
1452         rr = sldns_buffer_begin(scratch_buffer);
1453         rr_len = sldns_buffer_limit(scratch_buffer);
1454         dname_len = dname_valid(rr, rr_len);
1455         return az_remove_rr(z, rr, rr_len, dname_len, nonexist);
1456 }
1457
1458 /**
1459  * Parse zonefile
1460  * @param z: zone to read in.
1461  * @param in: file to read from (just opened).
1462  * @param rr: buffer to use for RRs, 64k.
1463  *      passed so that recursive includes can use the same buffer and do
1464  *      not grow the stack too much.
1465  * @param rrbuflen: sizeof rr buffer.
1466  * @param state: parse state with $ORIGIN, $TTL and 'prev-dname' and so on,
1467  *      that is kept between includes.
1468  *      The lineno is set at 1 and then increased by the function.
1469  * @param fname: file name.
1470  * @param depth: recursion depth for includes
1471  * @param cfg: config for chroot.
1472  * returns false on failure, has printed an error message
1473  */
1474 static int
1475 az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
1476         struct sldns_file_parse_state* state, char* fname, int depth,
1477         struct config_file* cfg)
1478 {
1479         size_t rr_len, dname_len;
1480         int status;
1481         state->lineno = 1;
1482
1483         while(!feof(in)) {
1484                 rr_len = rrbuflen;
1485                 dname_len = 0;
1486                 status = sldns_fp2wire_rr_buf(in, rr, &rr_len, &dname_len,
1487                         state);
1488                 if(status == LDNS_WIREPARSE_ERR_INCLUDE && rr_len == 0) {
1489                         /* we have $INCLUDE or $something */
1490                         if(strncmp((char*)rr, "$INCLUDE ", 9) == 0 ||
1491                            strncmp((char*)rr, "$INCLUDE\t", 9) == 0) {
1492                                 FILE* inc;
1493                                 int lineno_orig = state->lineno;
1494                                 char* incfile = (char*)rr + 8;
1495                                 if(depth > MAX_INCLUDE_DEPTH) {
1496                                         log_err("%s:%d max include depth"
1497                                           "exceeded", fname, state->lineno);
1498                                         return 0;
1499                                 }
1500                                 /* skip spaces */
1501                                 while(*incfile == ' ' || *incfile == '\t')
1502                                         incfile++;
1503                                 /* adjust for chroot on include file */
1504                                 if(cfg->chrootdir && cfg->chrootdir[0] &&
1505                                         strncmp(incfile, cfg->chrootdir,
1506                                                 strlen(cfg->chrootdir)) == 0)
1507                                         incfile += strlen(cfg->chrootdir);
1508                                 incfile = strdup(incfile);
1509                                 if(!incfile) {
1510                                         log_err("malloc failure");
1511                                         return 0;
1512                                 }
1513                                 verbose(VERB_ALGO, "opening $INCLUDE %s",
1514                                         incfile);
1515                                 inc = fopen(incfile, "r");
1516                                 if(!inc) {
1517                                         log_err("%s:%d cannot open include "
1518                                                 "file %s: %s", fname,
1519                                                 lineno_orig, incfile,
1520                                                 strerror(errno));
1521                                         free(incfile);
1522                                         return 0;
1523                                 }
1524                                 /* recurse read that file now */
1525                                 if(!az_parse_file(z, inc, rr, rrbuflen,
1526                                         state, incfile, depth+1, cfg)) {
1527                                         log_err("%s:%d cannot parse include "
1528                                                 "file %s", fname,
1529                                                 lineno_orig, incfile);
1530                                         fclose(inc);
1531                                         free(incfile);
1532                                         return 0;
1533                                 }
1534                                 fclose(inc);
1535                                 verbose(VERB_ALGO, "done with $INCLUDE %s",
1536                                         incfile);
1537                                 free(incfile);
1538                                 state->lineno = lineno_orig;
1539                         }
1540                         continue;
1541                 }
1542                 if(status != 0) {
1543                         log_err("parse error %s %d:%d: %s", fname,
1544                                 state->lineno, LDNS_WIREPARSE_OFFSET(status),
1545                                 sldns_get_errorstr_parse(status));
1546                         return 0;
1547                 }
1548                 if(rr_len == 0) {
1549                         /* EMPTY line, TTL or ORIGIN */
1550                         continue;
1551                 }
1552                 /* insert wirerr in rrbuf */
1553                 if(!az_insert_rr(z, rr, rr_len, dname_len, NULL)) {
1554                         char buf[17];
1555                         sldns_wire2str_type_buf(sldns_wirerr_get_type(rr,
1556                                 rr_len, dname_len), buf, sizeof(buf));
1557                         log_err("%s:%d cannot insert RR of type %s",
1558                                 fname, state->lineno, buf);
1559                         return 0;
1560                 }
1561         }
1562         return 1;
1563 }
1564
1565 int
1566 auth_zone_read_zonefile(struct auth_zone* z, struct config_file* cfg)
1567 {
1568         uint8_t rr[LDNS_RR_BUF_SIZE];
1569         struct sldns_file_parse_state state;
1570         char* zfilename;
1571         FILE* in;
1572         if(!z || !z->zonefile || z->zonefile[0]==0)
1573                 return 1; /* no file, or "", nothing to read */
1574
1575         zfilename = z->zonefile;
1576         if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
1577                 cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
1578                 zfilename += strlen(cfg->chrootdir);
1579         if(verbosity >= VERB_ALGO) {
1580                 char nm[255+1];
1581                 dname_str(z->name, nm);
1582                 verbose(VERB_ALGO, "read zonefile %s for %s", zfilename, nm);
1583         }
1584         in = fopen(zfilename, "r");
1585         if(!in) {
1586                 char* n = sldns_wire2str_dname(z->name, z->namelen);
1587                 if(z->zone_is_slave && errno == ENOENT) {
1588                         /* we fetch the zone contents later, no file yet */
1589                         verbose(VERB_ALGO, "no zonefile %s for %s",
1590                                 zfilename, n?n:"error");
1591                         free(n);
1592                         return 1;
1593                 }
1594                 log_err("cannot open zonefile %s for %s: %s",
1595                         zfilename, n?n:"error", strerror(errno));
1596                 free(n);
1597                 return 0;
1598         }
1599
1600         /* clear the data tree */
1601         traverse_postorder(&z->data, auth_data_del, NULL);
1602         rbtree_init(&z->data, &auth_data_cmp);
1603         /* clear the RPZ policies */
1604         if(z->rpz)
1605                 rpz_clear(z->rpz);
1606
1607         memset(&state, 0, sizeof(state));
1608         /* default TTL to 3600 */
1609         state.default_ttl = 3600;
1610         /* set $ORIGIN to the zone name */
1611         if(z->namelen <= sizeof(state.origin)) {
1612                 memcpy(state.origin, z->name, z->namelen);
1613                 state.origin_len = z->namelen;
1614         }
1615         /* parse the (toplevel) file */
1616         if(!az_parse_file(z, in, rr, sizeof(rr), &state, zfilename, 0, cfg)) {
1617                 char* n = sldns_wire2str_dname(z->name, z->namelen);
1618                 log_err("error parsing zonefile %s for %s",
1619                         zfilename, n?n:"error");
1620                 free(n);
1621                 fclose(in);
1622                 return 0;
1623         }
1624         fclose(in);
1625
1626         if(z->rpz)
1627                 rpz_finish_config(z->rpz);
1628         return 1;
1629 }
1630
1631 /** write buffer to file and check return codes */
1632 static int
1633 write_out(FILE* out, const char* str, size_t len)
1634 {
1635         size_t r;
1636         if(len == 0)
1637                 return 1;
1638         r = fwrite(str, 1, len, out);
1639         if(r == 0) {
1640                 log_err("write failed: %s", strerror(errno));
1641                 return 0;
1642         } else if(r < len) {
1643                 log_err("write failed: too short (disk full?)");
1644                 return 0;
1645         }
1646         return 1;
1647 }
1648
1649 /** convert auth rr to string */
1650 static int
1651 auth_rr_to_string(uint8_t* nm, size_t nmlen, uint16_t tp, uint16_t cl,
1652         struct packed_rrset_data* data, size_t i, char* s, size_t buflen)
1653 {
1654         int w = 0;
1655         size_t slen = buflen, datlen;
1656         uint8_t* dat;
1657         if(i >= data->count) tp = LDNS_RR_TYPE_RRSIG;
1658         dat = nm;
1659         datlen = nmlen;
1660         w += sldns_wire2str_dname_scan(&dat, &datlen, &s, &slen, NULL, 0, NULL);
1661         w += sldns_str_print(&s, &slen, "\t");
1662         w += sldns_str_print(&s, &slen, "%lu\t", (unsigned long)data->rr_ttl[i]);
1663         w += sldns_wire2str_class_print(&s, &slen, cl);
1664         w += sldns_str_print(&s, &slen, "\t");
1665         w += sldns_wire2str_type_print(&s, &slen, tp);
1666         w += sldns_str_print(&s, &slen, "\t");
1667         datlen = data->rr_len[i]-2;
1668         dat = data->rr_data[i]+2;
1669         w += sldns_wire2str_rdata_scan(&dat, &datlen, &s, &slen, tp, NULL, 0, NULL);
1670
1671         if(tp == LDNS_RR_TYPE_DNSKEY) {
1672                 w += sldns_str_print(&s, &slen, " ;{id = %u}",
1673                         sldns_calc_keytag_raw(data->rr_data[i]+2,
1674                                 data->rr_len[i]-2));
1675         }
1676         w += sldns_str_print(&s, &slen, "\n");
1677
1678         if(w >= (int)buflen) {
1679                 log_nametypeclass(NO_VERBOSE, "RR too long to print", nm, tp, cl);
1680                 return 0;
1681         }
1682         return 1;
1683 }
1684
1685 /** write rrset to file */
1686 static int
1687 auth_zone_write_rrset(struct auth_zone* z, struct auth_data* node,
1688         struct auth_rrset* r, FILE* out)
1689 {
1690         size_t i, count = r->data->count + r->data->rrsig_count;
1691         char buf[LDNS_RR_BUF_SIZE];
1692         for(i=0; i<count; i++) {
1693                 if(!auth_rr_to_string(node->name, node->namelen, r->type,
1694                         z->dclass, r->data, i, buf, sizeof(buf))) {
1695                         verbose(VERB_ALGO, "failed to rr2str rr %d", (int)i);
1696                         continue;
1697                 }
1698                 if(!write_out(out, buf, strlen(buf)))
1699                         return 0;
1700         }
1701         return 1;
1702 }
1703
1704 /** write domain to file */
1705 static int
1706 auth_zone_write_domain(struct auth_zone* z, struct auth_data* n, FILE* out)
1707 {
1708         struct auth_rrset* r;
1709         /* if this is zone apex, write SOA first */
1710         if(z->namelen == n->namelen) {
1711                 struct auth_rrset* soa = az_domain_rrset(n, LDNS_RR_TYPE_SOA);
1712                 if(soa) {
1713                         if(!auth_zone_write_rrset(z, n, soa, out))
1714                                 return 0;
1715                 }
1716         }
1717         /* write all the RRsets for this domain */
1718         for(r = n->rrsets; r; r = r->next) {
1719                 if(z->namelen == n->namelen &&
1720                         r->type == LDNS_RR_TYPE_SOA)
1721                         continue; /* skip SOA here */
1722                 if(!auth_zone_write_rrset(z, n, r, out))
1723                         return 0;
1724         }
1725         return 1;
1726 }
1727
1728 int auth_zone_write_file(struct auth_zone* z, const char* fname)
1729 {
1730         FILE* out;
1731         struct auth_data* n;
1732         out = fopen(fname, "w");
1733         if(!out) {
1734                 log_err("could not open %s: %s", fname, strerror(errno));
1735                 return 0;
1736         }
1737         RBTREE_FOR(n, struct auth_data*, &z->data) {
1738                 if(!auth_zone_write_domain(z, n, out)) {
1739                         log_err("could not write domain to %s", fname);
1740                         fclose(out);
1741                         return 0;
1742                 }
1743         }
1744         fclose(out);
1745         return 1;
1746 }
1747
1748 /** offline verify for zonemd, while reading a zone file to immediately
1749  * spot bad hashes in zonefile as they are read.
1750  * Creates temp buffers, but uses anchors and validation environment
1751  * from the module_env. */
1752 static void
1753 zonemd_offline_verify(struct auth_zone* z, struct module_env* env_for_val,
1754         struct module_stack* mods)
1755 {
1756         struct module_env env;
1757         time_t now = 0;
1758         if(!z->zonemd_check)
1759                 return;
1760         env = *env_for_val;
1761         env.scratch_buffer = sldns_buffer_new(env.cfg->msg_buffer_size);
1762         if(!env.scratch_buffer) {
1763                 log_err("out of memory");
1764                 goto clean_exit;
1765         }
1766         env.scratch = regional_create();
1767         if(!env.now) {
1768                 env.now = &now;
1769                 now = time(NULL);
1770         }
1771         if(!env.scratch) {
1772                 log_err("out of memory");
1773                 goto clean_exit;
1774         }
1775         auth_zone_verify_zonemd(z, &env, mods, NULL, 1, 0);
1776
1777 clean_exit:
1778         /* clean up and exit */
1779         sldns_buffer_free(env.scratch_buffer);
1780         regional_destroy(env.scratch);
1781 }
1782
1783 /** read all auth zones from file (if they have) */
1784 static int
1785 auth_zones_read_zones(struct auth_zones* az, struct config_file* cfg,
1786         struct module_env* env, struct module_stack* mods)
1787 {
1788         struct auth_zone* z;
1789         lock_rw_wrlock(&az->lock);
1790         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1791                 lock_rw_wrlock(&z->lock);
1792                 if(!auth_zone_read_zonefile(z, cfg)) {
1793                         lock_rw_unlock(&z->lock);
1794                         lock_rw_unlock(&az->lock);
1795                         return 0;
1796                 }
1797                 if(z->zonefile && z->zonefile[0]!=0 && env)
1798                         zonemd_offline_verify(z, env, mods);
1799                 lock_rw_unlock(&z->lock);
1800         }
1801         lock_rw_unlock(&az->lock);
1802         return 1;
1803 }
1804
1805 /** fetch the content of a ZONEMD RR from the rdata */
1806 static int zonemd_fetch_parameters(struct auth_rrset* zonemd_rrset, size_t i,
1807         uint32_t* serial, int* scheme, int* hashalgo, uint8_t** hash,
1808         size_t* hashlen)
1809 {
1810         size_t rr_len;
1811         uint8_t* rdata;
1812         if(i >= zonemd_rrset->data->count)
1813                 return 0;
1814         rr_len = zonemd_rrset->data->rr_len[i];
1815         if(rr_len < 2+4+1+1)
1816                 return 0; /* too short, for rdlen+serial+scheme+algo */
1817         rdata = zonemd_rrset->data->rr_data[i];
1818         *serial = sldns_read_uint32(rdata+2);
1819         *scheme = rdata[6];
1820         *hashalgo = rdata[7];
1821         *hashlen = rr_len - 8;
1822         if(*hashlen == 0)
1823                 *hash = NULL;
1824         else    *hash = rdata+8;
1825         return 1;
1826 }
1827
1828 /**
1829  * See if the ZONEMD scheme, hash occurs more than once.
1830  * @param zonemd_rrset: the zonemd rrset to check with the RRs in it.
1831  * @param index: index of the original, this is allowed to have that
1832  *      scheme and hashalgo, but other RRs should not have it.
1833  * @param scheme: the scheme to check for.
1834  * @param hashalgo: the hash algorithm to check for.
1835  * @return true if it occurs more than once.
1836  */
1837 static int zonemd_is_duplicate_scheme_hash(struct auth_rrset* zonemd_rrset,
1838         size_t index, int scheme, int hashalgo)
1839 {
1840         size_t j;
1841         for(j=0; j<zonemd_rrset->data->count; j++) {
1842                 uint32_t serial2 = 0;
1843                 int scheme2 = 0, hashalgo2 = 0;
1844                 uint8_t* hash2 = NULL;
1845                 size_t hashlen2 = 0;
1846                 if(index == j) {
1847                         /* this is the original */
1848                         continue;
1849                 }
1850                 if(!zonemd_fetch_parameters(zonemd_rrset, j, &serial2,
1851                         &scheme2, &hashalgo2, &hash2, &hashlen2)) {
1852                         /* malformed, skip it */
1853                         continue;
1854                 }
1855                 if(scheme == scheme2 && hashalgo == hashalgo2) {
1856                         /* duplicate scheme, hash */
1857                         verbose(VERB_ALGO, "zonemd duplicate for scheme %d "
1858                                 "and hash %d", scheme, hashalgo);
1859                         return 1;
1860                 }
1861         }
1862         return 0;
1863 }
1864
1865 /**
1866  * Check ZONEMDs if present for the auth zone.  Depending on config
1867  * it can warn or fail on that.  Checks the hash of the ZONEMD.
1868  * @param z: auth zone to check for.
1869  *      caller must hold lock on zone.
1870  * @param env: module env for temp buffers.
1871  * @param reason: returned on failure.
1872  * @return false on failure, true if hash checks out.
1873  */
1874 static int auth_zone_zonemd_check_hash(struct auth_zone* z,
1875         struct module_env* env, char** reason)
1876 {
1877         /* loop over ZONEMDs and see which one is valid. if not print
1878          * failure (depending on config) */
1879         struct auth_data* apex;
1880         struct auth_rrset* zonemd_rrset;
1881         size_t i;
1882         struct regional* region = NULL;
1883         struct sldns_buffer* buf = NULL;
1884         uint32_t soa_serial = 0;
1885         region = env->scratch;
1886         regional_free_all(region);
1887         buf = env->scratch_buffer;
1888         if(!auth_zone_get_serial(z, &soa_serial)) {
1889                 *reason = "zone has no SOA serial";
1890                 return 0;
1891         }
1892
1893         apex = az_find_name(z, z->name, z->namelen);
1894         if(!apex) {
1895                 *reason = "zone has no apex";
1896                 return 0;
1897         }
1898         zonemd_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_ZONEMD);
1899         if(!zonemd_rrset || zonemd_rrset->data->count==0) {
1900                 *reason = "zone has no ZONEMD";
1901                 return 0; /* no RRset or no RRs in rrset */
1902         }
1903
1904         /* we have a ZONEMD, check if it is correct */
1905         for(i=0; i<zonemd_rrset->data->count; i++) {
1906                 uint32_t serial = 0;
1907                 int scheme = 0, hashalgo = 0;
1908                 uint8_t* hash = NULL;
1909                 size_t hashlen = 0;
1910                 if(!zonemd_fetch_parameters(zonemd_rrset, i, &serial, &scheme,
1911                         &hashalgo, &hash, &hashlen)) {
1912                         /* malformed RR */
1913                         *reason = "ZONEMD rdata malformed";
1914                         continue;
1915                 }
1916                 /* check for duplicates */
1917                 if(zonemd_is_duplicate_scheme_hash(zonemd_rrset, i, scheme,
1918                         hashalgo)) {
1919                         /* duplicate hash of the same scheme,hash
1920                          * is not allowed. */
1921                         *reason = "ZONEMD RRSet contains more than one RR "
1922                                 "with the same scheme and hash algorithm";
1923                         continue;
1924                 }
1925                 regional_free_all(region);
1926                 if(serial != soa_serial) {
1927                         *reason = "ZONEMD serial is wrong";
1928                         continue;
1929                 }
1930                 if(auth_zone_generate_zonemd_check(z, scheme, hashalgo,
1931                         hash, hashlen, region, buf, reason)) {
1932                         /* success */
1933                         if(verbosity >= VERB_ALGO) {
1934                                 char zstr[255+1];
1935                                 dname_str(z->name, zstr);
1936                                 verbose(VERB_ALGO, "auth-zone %s ZONEMD hash is correct", zstr);
1937                         }
1938                         return 1;
1939                 }
1940                 /* try next one */
1941         }
1942         /* fail, we may have reason */
1943         if(!*reason)
1944                 *reason = "no ZONEMD records found";
1945         if(verbosity >= VERB_ALGO) {
1946                 char zstr[255+1];
1947                 dname_str(z->name, zstr);
1948                 verbose(VERB_ALGO, "auth-zone %s ZONEMD failed: %s", zstr, *reason);
1949         }
1950         return 0;
1951 }
1952
1953 /** find serial number of zone or false if none */
1954 int
1955 auth_zone_get_serial(struct auth_zone* z, uint32_t* serial)
1956 {
1957         struct auth_data* apex;
1958         struct auth_rrset* soa;
1959         struct packed_rrset_data* d;
1960         apex = az_find_name(z, z->name, z->namelen);
1961         if(!apex) return 0;
1962         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1963         if(!soa || soa->data->count==0)
1964                 return 0; /* no RRset or no RRs in rrset */
1965         if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1966         d = soa->data;
1967         *serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1968         return 1;
1969 }
1970
1971 /** Find auth_zone SOA and populate the values in xfr(soa values). */
1972 int
1973 xfr_find_soa(struct auth_zone* z, struct auth_xfer* xfr)
1974 {
1975         struct auth_data* apex;
1976         struct auth_rrset* soa;
1977         struct packed_rrset_data* d;
1978         apex = az_find_name(z, z->name, z->namelen);
1979         if(!apex) return 0;
1980         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1981         if(!soa || soa->data->count==0)
1982                 return 0; /* no RRset or no RRs in rrset */
1983         if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1984         /* SOA record ends with serial, refresh, retry, expiry, minimum,
1985          * as 4 byte fields */
1986         d = soa->data;
1987         xfr->have_zone = 1;
1988         xfr->serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1989         xfr->refresh = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-16));
1990         xfr->retry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-12));
1991         xfr->expiry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-8));
1992         /* soa minimum at d->rr_len[0]-4 */
1993         return 1;
1994 }
1995
1996 /**
1997  * Setup auth_xfer zone
1998  * This populates the have_zone, soa values, and so on times.
1999  * Doesn't do network traffic yet, can set option flags.
2000  * @param z: locked by caller, and modified for setup
2001  * @param x: locked by caller, and modified.
2002  * @return false on failure.
2003  */
2004 static int
2005 auth_xfer_setup(struct auth_zone* z, struct auth_xfer* x)
2006 {
2007         /* for a zone without zone transfers, x==NULL, so skip them,
2008          * i.e. the zone config is fixed with no masters or urls */
2009         if(!z || !x) return 1;
2010         if(!xfr_find_soa(z, x)) {
2011                 return 1;
2012         }
2013         /* nothing for probe, nextprobe and transfer tasks */
2014         return 1;
2015 }
2016
2017 /**
2018  * Setup all zones
2019  * @param az: auth zones structure
2020  * @return false on failure.
2021  */
2022 static int
2023 auth_zones_setup_zones(struct auth_zones* az)
2024 {
2025         struct auth_zone* z;
2026         struct auth_xfer* x;
2027         lock_rw_wrlock(&az->lock);
2028         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2029                 lock_rw_wrlock(&z->lock);
2030                 x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
2031                 if(x) {
2032                         lock_basic_lock(&x->lock);
2033                 }
2034                 if(!auth_xfer_setup(z, x)) {
2035                         if(x) {
2036                                 lock_basic_unlock(&x->lock);
2037                         }
2038                         lock_rw_unlock(&z->lock);
2039                         lock_rw_unlock(&az->lock);
2040                         return 0;
2041                 }
2042                 if(x) {
2043                         lock_basic_unlock(&x->lock);
2044                 }
2045                 lock_rw_unlock(&z->lock);
2046         }
2047         lock_rw_unlock(&az->lock);
2048         return 1;
2049 }
2050
2051 /** set config items and create zones */
2052 static int
2053 auth_zones_cfg(struct auth_zones* az, struct config_auth* c)
2054 {
2055         struct auth_zone* z;
2056         struct auth_xfer* x = NULL;
2057
2058         /* create zone */
2059         if(c->isrpz) {
2060                 /* if the rpz lock is needed, grab it before the other
2061                  * locks to avoid a lock dependency cycle */
2062                 lock_rw_wrlock(&az->rpz_lock);
2063         }
2064         lock_rw_wrlock(&az->lock);
2065         if(!(z=auth_zones_find_or_add_zone(az, c->name))) {
2066                 lock_rw_unlock(&az->lock);
2067                 if(c->isrpz) {
2068                         lock_rw_unlock(&az->rpz_lock);
2069                 }
2070                 return 0;
2071         }
2072         if(c->masters || c->urls) {
2073                 if(!(x=auth_zones_find_or_add_xfer(az, z))) {
2074                         lock_rw_unlock(&az->lock);
2075                         lock_rw_unlock(&z->lock);
2076                         if(c->isrpz) {
2077                                 lock_rw_unlock(&az->rpz_lock);
2078                         }
2079                         return 0;
2080                 }
2081         }
2082         if(c->for_downstream)
2083                 az->have_downstream = 1;
2084         lock_rw_unlock(&az->lock);
2085
2086         /* set options */
2087         z->zone_deleted = 0;
2088         if(!auth_zone_set_zonefile(z, c->zonefile)) {
2089                 if(x) {
2090                         lock_basic_unlock(&x->lock);
2091                 }
2092                 lock_rw_unlock(&z->lock);
2093                 if(c->isrpz) {
2094                         lock_rw_unlock(&az->rpz_lock);
2095                 }
2096                 return 0;
2097         }
2098         z->for_downstream = c->for_downstream;
2099         z->for_upstream = c->for_upstream;
2100         z->fallback_enabled = c->fallback_enabled;
2101         z->zonemd_check = c->zonemd_check;
2102         z->zonemd_reject_absence = c->zonemd_reject_absence;
2103         if(c->isrpz && !z->rpz){
2104                 if(!(z->rpz = rpz_create(c))){
2105                         fatal_exit("Could not setup RPZ zones");
2106                         return 0;
2107                 }
2108                 lock_protect(&z->lock, &z->rpz->local_zones, sizeof(*z->rpz));
2109                 /* the az->rpz_lock is locked above */
2110                 z->rpz_az_next = az->rpz_first;
2111                 if(az->rpz_first)
2112                         az->rpz_first->rpz_az_prev = z;
2113                 az->rpz_first = z;
2114         }
2115         if(c->isrpz) {
2116                 lock_rw_unlock(&az->rpz_lock);
2117         }
2118
2119         /* xfer zone */
2120         if(x) {
2121                 z->zone_is_slave = 1;
2122                 /* set options on xfer zone */
2123                 if(!xfer_set_masters(&x->task_probe->masters, c, 0)) {
2124                         lock_basic_unlock(&x->lock);
2125                         lock_rw_unlock(&z->lock);
2126                         return 0;
2127                 }
2128                 if(!xfer_set_masters(&x->task_transfer->masters, c, 1)) {
2129                         lock_basic_unlock(&x->lock);
2130                         lock_rw_unlock(&z->lock);
2131                         return 0;
2132                 }
2133                 lock_basic_unlock(&x->lock);
2134         }
2135
2136         lock_rw_unlock(&z->lock);
2137         return 1;
2138 }
2139
2140 /** set all auth zones deleted, then in auth_zones_cfg, it marks them
2141  * as nondeleted (if they are still in the config), and then later
2142  * we can find deleted zones */
2143 static void
2144 az_setall_deleted(struct auth_zones* az)
2145 {
2146         struct auth_zone* z;
2147         lock_rw_wrlock(&az->lock);
2148         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2149                 lock_rw_wrlock(&z->lock);
2150                 z->zone_deleted = 1;
2151                 lock_rw_unlock(&z->lock);
2152         }
2153         lock_rw_unlock(&az->lock);
2154 }
2155
2156 /** find zones that are marked deleted and delete them.
2157  * This is called from apply_cfg, and there are no threads and no
2158  * workers, so the xfr can just be deleted. */
2159 static void
2160 az_delete_deleted_zones(struct auth_zones* az)
2161 {
2162         struct auth_zone* z;
2163         struct auth_zone* delete_list = NULL, *next;
2164         struct auth_xfer* xfr;
2165         lock_rw_wrlock(&az->lock);
2166         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2167                 lock_rw_wrlock(&z->lock);
2168                 if(z->zone_deleted) {
2169                         /* we cannot alter the rbtree right now, but
2170                          * we can put it on a linked list and then
2171                          * delete it */
2172                         z->delete_next = delete_list;
2173                         delete_list = z;
2174                 }
2175                 lock_rw_unlock(&z->lock);
2176         }
2177         /* now we are out of the tree loop and we can loop and delete
2178          * the zones */
2179         z = delete_list;
2180         while(z) {
2181                 next = z->delete_next;
2182                 xfr = auth_xfer_find(az, z->name, z->namelen, z->dclass);
2183                 if(xfr) {
2184                         (void)rbtree_delete(&az->xtree, &xfr->node);
2185                         auth_xfer_delete(xfr);
2186                 }
2187                 (void)rbtree_delete(&az->ztree, &z->node);
2188                 auth_zone_delete(z, az);
2189                 z = next;
2190         }
2191         lock_rw_unlock(&az->lock);
2192 }
2193
2194 int auth_zones_apply_cfg(struct auth_zones* az, struct config_file* cfg,
2195         int setup, int* is_rpz, struct module_env* env,
2196         struct module_stack* mods)
2197 {
2198         struct config_auth* p;
2199         az_setall_deleted(az);
2200         for(p = cfg->auths; p; p = p->next) {
2201                 if(!p->name || p->name[0] == 0) {
2202                         log_warn("auth-zone without a name, skipped");
2203                         continue;
2204                 }
2205                 *is_rpz = (*is_rpz || p->isrpz);
2206                 if(!auth_zones_cfg(az, p)) {
2207                         log_err("cannot config auth zone %s", p->name);
2208                         return 0;
2209                 }
2210         }
2211         az_delete_deleted_zones(az);
2212         if(!auth_zones_read_zones(az, cfg, env, mods))
2213                 return 0;
2214         if(setup) {
2215                 if(!auth_zones_setup_zones(az))
2216                         return 0;
2217         }
2218         return 1;
2219 }
2220
2221 /** delete chunks
2222  * @param at: transfer structure with chunks list.  The chunks and their
2223  *      data are freed.
2224  */
2225 static void
2226 auth_chunks_delete(struct auth_transfer* at)
2227 {
2228         if(at->chunks_first) {
2229                 struct auth_chunk* c, *cn;
2230                 c = at->chunks_first;
2231                 while(c) {
2232                         cn = c->next;
2233                         free(c->data);
2234                         free(c);
2235                         c = cn;
2236                 }
2237         }
2238         at->chunks_first = NULL;
2239         at->chunks_last = NULL;
2240 }
2241
2242 /** free master addr list */
2243 static void
2244 auth_free_master_addrs(struct auth_addr* list)
2245 {
2246         struct auth_addr *n;
2247         while(list) {
2248                 n = list->next;
2249                 free(list);
2250                 list = n;
2251         }
2252 }
2253
2254 /** free the masters list */
2255 static void
2256 auth_free_masters(struct auth_master* list)
2257 {
2258         struct auth_master* n;
2259         while(list) {
2260                 n = list->next;
2261                 auth_free_master_addrs(list->list);
2262                 free(list->host);
2263                 free(list->file);
2264                 free(list);
2265                 list = n;
2266         }
2267 }
2268
2269 /** delete auth xfer structure
2270  * @param xfr: delete this xfer and its tasks.
2271  */
2272 static void
2273 auth_xfer_delete(struct auth_xfer* xfr)
2274 {
2275         if(!xfr) return;
2276         lock_basic_destroy(&xfr->lock);
2277         free(xfr->name);
2278         if(xfr->task_nextprobe) {
2279                 comm_timer_delete(xfr->task_nextprobe->timer);
2280                 free(xfr->task_nextprobe);
2281         }
2282         if(xfr->task_probe) {
2283                 auth_free_masters(xfr->task_probe->masters);
2284                 comm_point_delete(xfr->task_probe->cp);
2285                 comm_timer_delete(xfr->task_probe->timer);
2286                 free(xfr->task_probe);
2287         }
2288         if(xfr->task_transfer) {
2289                 auth_free_masters(xfr->task_transfer->masters);
2290                 comm_point_delete(xfr->task_transfer->cp);
2291                 comm_timer_delete(xfr->task_transfer->timer);
2292                 if(xfr->task_transfer->chunks_first) {
2293                         auth_chunks_delete(xfr->task_transfer);
2294                 }
2295                 free(xfr->task_transfer);
2296         }
2297         auth_free_masters(xfr->allow_notify_list);
2298         free(xfr);
2299 }
2300
2301 /** helper traverse to delete zones */
2302 static void
2303 auth_zone_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2304 {
2305         struct auth_zone* z = (struct auth_zone*)n->key;
2306         auth_zone_delete(z, NULL);
2307 }
2308
2309 /** helper traverse to delete xfer zones */
2310 static void
2311 auth_xfer_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2312 {
2313         struct auth_xfer* z = (struct auth_xfer*)n->key;
2314         auth_xfer_delete(z);
2315 }
2316
2317 void auth_zones_delete(struct auth_zones* az)
2318 {
2319         if(!az) return;
2320         lock_rw_destroy(&az->lock);
2321         lock_rw_destroy(&az->rpz_lock);
2322         traverse_postorder(&az->ztree, auth_zone_del, NULL);
2323         traverse_postorder(&az->xtree, auth_xfer_del, NULL);
2324         free(az);
2325 }
2326
2327 /** true if domain has only nsec3 */
2328 static int
2329 domain_has_only_nsec3(struct auth_data* n)
2330 {
2331         struct auth_rrset* rrset = n->rrsets;
2332         int nsec3_seen = 0;
2333         while(rrset) {
2334                 if(rrset->type == LDNS_RR_TYPE_NSEC3) {
2335                         nsec3_seen = 1;
2336                 } else if(rrset->type != LDNS_RR_TYPE_RRSIG) {
2337                         return 0;
2338                 }
2339                 rrset = rrset->next;
2340         }
2341         return nsec3_seen;
2342 }
2343
2344 /** see if the domain has a wildcard child '*.domain' */
2345 static struct auth_data*
2346 az_find_wildcard_domain(struct auth_zone* z, uint8_t* nm, size_t nmlen)
2347 {
2348         uint8_t wc[LDNS_MAX_DOMAINLEN];
2349         if(nmlen+2 > sizeof(wc))
2350                 return NULL; /* result would be too long */
2351         wc[0] = 1; /* length of wildcard label */
2352         wc[1] = (uint8_t)'*'; /* wildcard label */
2353         memmove(wc+2, nm, nmlen);
2354         return az_find_name(z, wc, nmlen+2);
2355 }
2356
2357 /** find wildcard between qname and cename */
2358 static struct auth_data*
2359 az_find_wildcard(struct auth_zone* z, struct query_info* qinfo,
2360         struct auth_data* ce)
2361 {
2362         uint8_t* nm = qinfo->qname;
2363         size_t nmlen = qinfo->qname_len;
2364         struct auth_data* node;
2365         if(!dname_subdomain_c(nm, z->name))
2366                 return NULL; /* out of zone */
2367         while((node=az_find_wildcard_domain(z, nm, nmlen))==NULL) {
2368                 /* see if we can go up to find the wildcard */
2369                 if(nmlen == z->namelen)
2370                         return NULL; /* top of zone reached */
2371                 if(ce && nmlen == ce->namelen)
2372                         return NULL; /* ce reached */
2373                 if(dname_is_root(nm))
2374                         return NULL; /* cannot go up */
2375                 dname_remove_label(&nm, &nmlen);
2376         }
2377         return node;
2378 }
2379
2380 /** domain is not exact, find first candidate ce (name that matches
2381  * a part of qname) in tree */
2382 static struct auth_data*
2383 az_find_candidate_ce(struct auth_zone* z, struct query_info* qinfo,
2384         struct auth_data* n)
2385 {
2386         uint8_t* nm;
2387         size_t nmlen;
2388         if(n) {
2389                 nm = dname_get_shared_topdomain(qinfo->qname, n->name);
2390         } else {
2391                 nm = qinfo->qname;
2392         }
2393         dname_count_size_labels(nm, &nmlen);
2394         n = az_find_name(z, nm, nmlen);
2395         /* delete labels and go up on name */
2396         while(!n) {
2397                 if(dname_is_root(nm))
2398                         return NULL; /* cannot go up */
2399                 dname_remove_label(&nm, &nmlen);
2400                 n = az_find_name(z, nm, nmlen);
2401         }
2402         return n;
2403 }
2404
2405 /** go up the auth tree to next existing name. */
2406 static struct auth_data*
2407 az_domain_go_up(struct auth_zone* z, struct auth_data* n)
2408 {
2409         uint8_t* nm = n->name;
2410         size_t nmlen = n->namelen;
2411         while(!dname_is_root(nm)) {
2412                 dname_remove_label(&nm, &nmlen);
2413                 if((n=az_find_name(z, nm, nmlen)) != NULL)
2414                         return n;
2415         }
2416         return NULL;
2417 }
2418
2419 /** Find the closest encloser, an name that exists and is above the
2420  * qname.
2421  * return true if the node (param node) is existing, nonobscured and
2422  *      can be used to generate answers from.  It is then also node_exact.
2423  * returns false if the node is not good enough (or it wasn't node_exact)
2424  *      in this case the ce can be filled.
2425  *      if ce is NULL, no ce exists, and likely the zone is completely empty,
2426  *      not even with a zone apex.
2427  *      if ce is nonNULL it is the closest enclosing upper name (that exists
2428  *      itself for answer purposes).  That name may have DNAME, NS or wildcard
2429  *      rrset is the closest DNAME or NS rrset that was found.
2430  */
2431 static int
2432 az_find_ce(struct auth_zone* z, struct query_info* qinfo,
2433         struct auth_data* node, int node_exact, struct auth_data** ce,
2434         struct auth_rrset** rrset)
2435 {
2436         struct auth_data* n = node;
2437         *ce = NULL;
2438         *rrset = NULL;
2439         if(!node_exact) {
2440                 /* if not exact, lookup closest exact match */
2441                 n = az_find_candidate_ce(z, qinfo, n);
2442         } else {
2443                 /* if exact, the node itself is the first candidate ce */
2444                 *ce = n;
2445         }
2446
2447         /* no direct answer from nsec3-only domains */
2448         if(n && domain_has_only_nsec3(n)) {
2449                 node_exact = 0;
2450                 *ce = NULL;
2451         }
2452
2453         /* with exact matches, walk up the labels until we find the
2454          * delegation, or DNAME or zone end */
2455         while(n) {
2456                 /* see if the current candidate has issues */
2457                 /* not zone apex and has type NS */
2458                 if(n->namelen != z->namelen &&
2459                         (*rrset=az_domain_rrset(n, LDNS_RR_TYPE_NS)) &&
2460                         /* delegate here, but DS at exact the dp has notype */
2461                         (qinfo->qtype != LDNS_RR_TYPE_DS ||
2462                         n->namelen != qinfo->qname_len)) {
2463                         /* referral */
2464                         /* this is ce and the lowernode is nonexisting */
2465                         *ce = n;
2466                         return 0;
2467                 }
2468                 /* not equal to qname and has type DNAME */
2469                 if(n->namelen != qinfo->qname_len &&
2470                         (*rrset=az_domain_rrset(n, LDNS_RR_TYPE_DNAME))) {
2471                         /* this is ce and the lowernode is nonexisting */
2472                         *ce = n;
2473                         return 0;
2474                 }
2475
2476                 if(*ce == NULL && !domain_has_only_nsec3(n)) {
2477                         /* if not found yet, this exact name must be
2478                          * our lowest match (but not nsec3onlydomain) */
2479                         *ce = n;
2480                 }
2481
2482                 /* walk up the tree by removing labels from name and lookup */
2483                 n = az_domain_go_up(z, n);
2484         }
2485         /* found no problems, if it was an exact node, it is fine to use */
2486         return node_exact;
2487 }
2488
2489 /** add additional A/AAAA from domain names in rrset rdata (+offset)
2490  * offset is number of bytes in rdata where the dname is located. */
2491 static int
2492 az_add_additionals_from(struct auth_zone* z, struct regional* region,
2493         struct dns_msg* msg, struct auth_rrset* rrset, size_t offset)
2494 {
2495         struct packed_rrset_data* d = rrset->data;
2496         size_t i;
2497         if(!d) return 0;
2498         for(i=0; i<d->count; i++) {
2499                 size_t dlen;
2500                 struct auth_data* domain;
2501                 struct auth_rrset* ref;
2502                 if(d->rr_len[i] < 2+offset)
2503                         continue; /* too short */
2504                 if(!(dlen = dname_valid(d->rr_data[i]+2+offset,
2505                         d->rr_len[i]-2-offset)))
2506                         continue; /* malformed */
2507                 domain = az_find_name(z, d->rr_data[i]+2+offset, dlen);
2508                 if(!domain)
2509                         continue;
2510                 if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_A)) != NULL) {
2511                         if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2512                                 return 0;
2513                 }
2514                 if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_AAAA)) != NULL) {
2515                         if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2516                                 return 0;
2517                 }
2518         }
2519         return 1;
2520 }
2521
2522 /** add negative SOA record (with negative TTL) */
2523 static int
2524 az_add_negative_soa(struct auth_zone* z, struct regional* region,
2525         struct dns_msg* msg)
2526 {
2527         time_t minimum;
2528         size_t i;
2529         struct packed_rrset_data* d;
2530         struct auth_rrset* soa;
2531         struct auth_data* apex = az_find_name(z, z->name, z->namelen);
2532         if(!apex) return 0;
2533         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
2534         if(!soa) return 0;
2535         /* must be first to put in message; we want to fix the TTL with
2536          * one RRset here, otherwise we'd need to loop over the RRs to get
2537          * the resulting lower TTL */
2538         log_assert(msg->rep->rrset_count == 0);
2539         if(!msg_add_rrset_ns(z, region, msg, apex, soa)) return 0;
2540         /* fixup TTL */
2541         d = (struct packed_rrset_data*)msg->rep->rrsets[msg->rep->rrset_count-1]->entry.data;
2542         /* last 4 bytes are minimum ttl in network format */
2543         if(d->count == 0) return 0;
2544         if(d->rr_len[0] < 2+4) return 0;
2545         minimum = (time_t)sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-4));
2546         minimum = d->ttl<minimum?d->ttl:minimum;
2547         d->ttl = minimum;
2548         for(i=0; i < d->count + d->rrsig_count; i++)
2549                 d->rr_ttl[i] = minimum;
2550         msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
2551         msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
2552         msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
2553         return 1;
2554 }
2555
2556 /** See if the query goes to empty nonterminal (that has no auth_data,
2557  * but there are nodes underneath.  We already checked that there are
2558  * not NS, or DNAME above, so that we only need to check if some node
2559  * exists below (with nonempty rr list), return true if emptynonterminal */
2560 static int
2561 az_empty_nonterminal(struct auth_zone* z, struct query_info* qinfo,
2562         struct auth_data* node)
2563 {
2564         struct auth_data* next;
2565         if(!node) {
2566                 /* no smaller was found, use first (smallest) node as the
2567                  * next one */
2568                 next = (struct auth_data*)rbtree_first(&z->data);
2569         } else {
2570                 next = (struct auth_data*)rbtree_next(&node->node);
2571         }
2572         while(next && (rbnode_type*)next != RBTREE_NULL && next->rrsets == NULL) {
2573                 /* the next name has empty rrsets, is an empty nonterminal
2574                  * itself, see if there exists something below it */
2575                 next = (struct auth_data*)rbtree_next(&node->node);
2576         }
2577         if((rbnode_type*)next == RBTREE_NULL || !next) {
2578                 /* there is no next node, so something below it cannot
2579                  * exist */
2580                 return 0;
2581         }
2582         /* a next node exists, if there was something below the query,
2583          * this node has to be it.  See if it is below the query name */
2584         if(dname_strict_subdomain_c(next->name, qinfo->qname))
2585                 return 1;
2586         return 0;
2587 }
2588
2589 /** create synth cname target name in buffer, or fail if too long */
2590 static size_t
2591 synth_cname_buf(uint8_t* qname, size_t qname_len, size_t dname_len,
2592         uint8_t* dtarg, size_t dtarglen, uint8_t* buf, size_t buflen)
2593 {
2594         size_t newlen = qname_len + dtarglen - dname_len;
2595         if(newlen > buflen) {
2596                 /* YXDOMAIN error */
2597                 return 0;
2598         }
2599         /* new name is concatenation of qname front (without DNAME owner)
2600          * and DNAME target name */
2601         memcpy(buf, qname, qname_len-dname_len);
2602         memmove(buf+(qname_len-dname_len), dtarg, dtarglen);
2603         return newlen;
2604 }
2605
2606 /** create synthetic CNAME rrset for in a DNAME answer in region,
2607  * false on alloc failure, cname==NULL when name too long. */
2608 static int
2609 create_synth_cname(uint8_t* qname, size_t qname_len, struct regional* region,
2610         struct auth_data* node, struct auth_rrset* dname, uint16_t dclass,
2611         struct ub_packed_rrset_key** cname)
2612 {
2613         uint8_t buf[LDNS_MAX_DOMAINLEN];
2614         uint8_t* dtarg;
2615         size_t dtarglen, newlen;
2616         struct packed_rrset_data* d;
2617
2618         /* get DNAME target name */
2619         if(dname->data->count < 1) return 0;
2620         if(dname->data->rr_len[0] < 3) return 0; /* at least rdatalen +1 */
2621         dtarg = dname->data->rr_data[0]+2;
2622         dtarglen = dname->data->rr_len[0]-2;
2623         if(sldns_read_uint16(dname->data->rr_data[0]) != dtarglen)
2624                 return 0; /* rdatalen in DNAME rdata is malformed */
2625         if(dname_valid(dtarg, dtarglen) != dtarglen)
2626                 return 0; /* DNAME RR has malformed rdata */
2627         if(qname_len == 0)
2628                 return 0; /* too short */
2629         if(qname_len <= node->namelen)
2630                 return 0; /* qname too short for dname removal */
2631
2632         /* synthesize a CNAME */
2633         newlen = synth_cname_buf(qname, qname_len, node->namelen,
2634                 dtarg, dtarglen, buf, sizeof(buf));
2635         if(newlen == 0) {
2636                 /* YXDOMAIN error */
2637                 *cname = NULL;
2638                 return 1;
2639         }
2640         *cname = (struct ub_packed_rrset_key*)regional_alloc(region,
2641                 sizeof(struct ub_packed_rrset_key));
2642         if(!*cname)
2643                 return 0; /* out of memory */
2644         memset(&(*cname)->entry, 0, sizeof((*cname)->entry));
2645         (*cname)->entry.key = (*cname);
2646         (*cname)->rk.type = htons(LDNS_RR_TYPE_CNAME);
2647         (*cname)->rk.rrset_class = htons(dclass);
2648         (*cname)->rk.flags = 0;
2649         (*cname)->rk.dname = regional_alloc_init(region, qname, qname_len);
2650         if(!(*cname)->rk.dname)
2651                 return 0; /* out of memory */
2652         (*cname)->rk.dname_len = qname_len;
2653         (*cname)->entry.hash = rrset_key_hash(&(*cname)->rk);
2654         d = (struct packed_rrset_data*)regional_alloc_zero(region,
2655                 sizeof(struct packed_rrset_data) + sizeof(size_t) +
2656                 sizeof(uint8_t*) + sizeof(time_t) + sizeof(uint16_t)
2657                 + newlen);
2658         if(!d)
2659                 return 0; /* out of memory */
2660         (*cname)->entry.data = d;
2661         d->ttl = 0; /* 0 for synthesized CNAME TTL */
2662         d->count = 1;
2663         d->rrsig_count = 0;
2664         d->trust = rrset_trust_ans_noAA;
2665         d->rr_len = (size_t*)((uint8_t*)d +
2666                 sizeof(struct packed_rrset_data));
2667         d->rr_len[0] = newlen + sizeof(uint16_t);
2668         packed_rrset_ptr_fixup(d);
2669         d->rr_ttl[0] = d->ttl;
2670         sldns_write_uint16(d->rr_data[0], newlen);
2671         memmove(d->rr_data[0] + sizeof(uint16_t), buf, newlen);
2672         return 1;
2673 }
2674
2675 /** add a synthesized CNAME to the answer section */
2676 static int
2677 add_synth_cname(struct auth_zone* z, uint8_t* qname, size_t qname_len,
2678         struct regional* region, struct dns_msg* msg, struct auth_data* dname,
2679         struct auth_rrset* rrset)
2680 {
2681         struct ub_packed_rrset_key* cname;
2682         /* synthesize a CNAME */
2683         if(!create_synth_cname(qname, qname_len, region, dname, rrset,
2684                 z->dclass, &cname)) {
2685                 /* out of memory */
2686                 return 0;
2687         }
2688         if(!cname) {
2689                 /* cname cannot be create because of YXDOMAIN */
2690                 msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
2691                 return 1;
2692         }
2693         /* add cname to message */
2694         if(!msg_grow_array(region, msg))
2695                 return 0;
2696         msg->rep->rrsets[msg->rep->rrset_count] = cname;
2697         msg->rep->rrset_count++;
2698         msg->rep->an_numrrsets++;
2699         msg_ttl(msg);
2700         return 1;
2701 }
2702
2703 /** Change a dname to a different one, for wildcard namechange */
2704 static void
2705 az_change_dnames(struct dns_msg* msg, uint8_t* oldname, uint8_t* newname,
2706         size_t newlen, int an_only)
2707 {
2708         size_t i;
2709         size_t start = 0, end = msg->rep->rrset_count;
2710         if(!an_only) start = msg->rep->an_numrrsets;
2711         if(an_only) end = msg->rep->an_numrrsets;
2712         for(i=start; i<end; i++) {
2713                 /* allocated in region so we can change the ptrs */
2714                 if(query_dname_compare(msg->rep->rrsets[i]->rk.dname, oldname)
2715                         == 0) {
2716                         msg->rep->rrsets[i]->rk.dname = newname;
2717                         msg->rep->rrsets[i]->rk.dname_len = newlen;
2718                 }
2719         }
2720 }
2721
2722 /** find NSEC record covering the query */
2723 static struct auth_rrset*
2724 az_find_nsec_cover(struct auth_zone* z, struct auth_data** node)
2725 {
2726         uint8_t* nm = (*node)->name;
2727         size_t nmlen = (*node)->namelen;
2728         struct auth_rrset* rrset;
2729         /* find the NSEC for the smallest-or-equal node */
2730         /* if node == NULL, we did not find a smaller name.  But the zone
2731          * name is the smallest name and should have an NSEC. So there is
2732          * no NSEC to return (for a properly signed zone) */
2733         /* for empty nonterminals, the auth-data node should not exist,
2734          * and thus we don't need to go rbtree_previous here to find
2735          * a domain with an NSEC record */
2736         /* but there could be glue, and if this is node, then it has no NSEC.
2737          * Go up to find nonglue (previous) NSEC-holding nodes */
2738         while((rrset=az_domain_rrset(*node, LDNS_RR_TYPE_NSEC)) == NULL) {
2739                 if(dname_is_root(nm)) return NULL;
2740                 if(nmlen == z->namelen) return NULL;
2741                 dname_remove_label(&nm, &nmlen);
2742                 /* adjust *node for the nsec rrset to find in */
2743                 *node = az_find_name(z, nm, nmlen);
2744         }
2745         return rrset;
2746 }
2747
2748 /** Find NSEC and add for wildcard denial */
2749 static int
2750 az_nsec_wildcard_denial(struct auth_zone* z, struct regional* region,
2751         struct dns_msg* msg, uint8_t* cenm, size_t cenmlen)
2752 {
2753         struct query_info qinfo;
2754         int node_exact;
2755         struct auth_data* node;
2756         struct auth_rrset* nsec;
2757         uint8_t wc[LDNS_MAX_DOMAINLEN];
2758         if(cenmlen+2 > sizeof(wc))
2759                 return 0; /* result would be too long */
2760         wc[0] = 1; /* length of wildcard label */
2761         wc[1] = (uint8_t)'*'; /* wildcard label */
2762         memmove(wc+2, cenm, cenmlen);
2763
2764         /* we have '*.ce' in wc wildcard name buffer */
2765         /* get nsec cover for that */
2766         qinfo.qname = wc;
2767         qinfo.qname_len = cenmlen+2;
2768         qinfo.qtype = 0;
2769         qinfo.qclass = 0;
2770         az_find_domain(z, &qinfo, &node_exact, &node);
2771         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
2772                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
2773         }
2774         return 1;
2775 }
2776
2777 /** Find the NSEC3PARAM rrset (if any) and if true you have the parameters */
2778 static int
2779 az_nsec3_param(struct auth_zone* z, int* algo, size_t* iter, uint8_t** salt,
2780         size_t* saltlen)
2781 {
2782         struct auth_data* apex;
2783         struct auth_rrset* param;
2784         size_t i;
2785         apex = az_find_name(z, z->name, z->namelen);
2786         if(!apex) return 0;
2787         param = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC3PARAM);
2788         if(!param || param->data->count==0)
2789                 return 0; /* no RRset or no RRs in rrset */
2790         /* find out which NSEC3PARAM RR has supported parameters */
2791         /* skip unknown flags (dynamic signer is recalculating nsec3 chain) */
2792         for(i=0; i<param->data->count; i++) {
2793                 uint8_t* rdata = param->data->rr_data[i]+2;
2794                 size_t rdatalen = param->data->rr_len[i];
2795                 if(rdatalen < 2+5)
2796                         continue; /* too short */
2797                 if(!nsec3_hash_algo_size_supported((int)(rdata[0])))
2798                         continue; /* unsupported algo */
2799                 if(rdatalen < (size_t)(2+5+(size_t)rdata[4]))
2800                         continue; /* salt missing */
2801                 if((rdata[1]&NSEC3_UNKNOWN_FLAGS)!=0)
2802                         continue; /* unknown flags */
2803                 *algo = (int)(rdata[0]);
2804                 *iter = sldns_read_uint16(rdata+2);
2805                 *saltlen = rdata[4];
2806                 if(*saltlen == 0)
2807                         *salt = NULL;
2808                 else    *salt = rdata+5;
2809                 return 1;
2810         }
2811         /* no supported params */
2812         return 0;
2813 }
2814
2815 /** Hash a name with nsec3param into buffer, it has zone name appended.
2816  * return length of hash */
2817 static size_t
2818 az_nsec3_hash(uint8_t* buf, size_t buflen, uint8_t* nm, size_t nmlen,
2819         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2820 {
2821         size_t hlen = nsec3_hash_algo_size_supported(algo);
2822         /* buffer has domain name, nsec3hash, and 256 is for max saltlen
2823          * (salt has 0-255 length) */
2824         unsigned char p[LDNS_MAX_DOMAINLEN+1+N3HASHBUFLEN+256];
2825         size_t i;
2826         if(nmlen+saltlen > sizeof(p) || hlen+saltlen > sizeof(p))
2827                 return 0;
2828         if(hlen > buflen)
2829                 return 0; /* somehow too large for destination buffer */
2830         /* hashfunc(name, salt) */
2831         memmove(p, nm, nmlen);
2832         query_dname_tolower(p);
2833         if(salt && saltlen > 0)
2834                 memmove(p+nmlen, salt, saltlen);
2835         (void)secalgo_nsec3_hash(algo, p, nmlen+saltlen, (unsigned char*)buf);
2836         for(i=0; i<iter; i++) {
2837                 /* hashfunc(hash, salt) */
2838                 memmove(p, buf, hlen);
2839                 if(salt && saltlen > 0)
2840                         memmove(p+hlen, salt, saltlen);
2841                 (void)secalgo_nsec3_hash(algo, p, hlen+saltlen,
2842                         (unsigned char*)buf);
2843         }
2844         return hlen;
2845 }
2846
2847 /** Hash name and return b32encoded hashname for lookup, zone name appended */
2848 static int
2849 az_nsec3_hashname(struct auth_zone* z, uint8_t* hashname, size_t* hashnmlen,
2850         uint8_t* nm, size_t nmlen, int algo, size_t iter, uint8_t* salt,
2851         size_t saltlen)
2852 {
2853         uint8_t hash[N3HASHBUFLEN];
2854         size_t hlen;
2855         int ret;
2856         hlen = az_nsec3_hash(hash, sizeof(hash), nm, nmlen, algo, iter,
2857                 salt, saltlen);
2858         if(!hlen) return 0;
2859         /* b32 encode */
2860         if(*hashnmlen < hlen*2+1+z->namelen) /* approx b32 as hexb16 */
2861                 return 0;
2862         ret = sldns_b32_ntop_extended_hex(hash, hlen, (char*)(hashname+1),
2863                 (*hashnmlen)-1);
2864         if(ret<1)
2865                 return 0;
2866         hashname[0] = (uint8_t)ret;
2867         ret++;
2868         if((*hashnmlen) - ret < z->namelen)
2869                 return 0;
2870         memmove(hashname+ret, z->name, z->namelen);
2871         *hashnmlen = z->namelen+(size_t)ret;
2872         return 1;
2873 }
2874
2875 /** Find the datanode that covers the nsec3hash-name */
2876 static struct auth_data*
2877 az_nsec3_findnode(struct auth_zone* z, uint8_t* hashnm, size_t hashnmlen)
2878 {
2879         struct query_info qinfo;
2880         struct auth_data* node;
2881         int node_exact;
2882         qinfo.qclass = 0;
2883         qinfo.qtype = 0;
2884         qinfo.qname = hashnm;
2885         qinfo.qname_len = hashnmlen;
2886         /* because canonical ordering and b32 nsec3 ordering are the same.
2887          * this is a good lookup to find the nsec3 name. */
2888         az_find_domain(z, &qinfo, &node_exact, &node);
2889         /* but we may have to skip non-nsec3 nodes */
2890         /* this may be a lot, the way to speed that up is to have a
2891          * separate nsec3 tree with nsec3 nodes */
2892         while(node && (rbnode_type*)node != RBTREE_NULL &&
2893                 !az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2894                 node = (struct auth_data*)rbtree_previous(&node->node);
2895         }
2896         if((rbnode_type*)node == RBTREE_NULL)
2897                 node = NULL;
2898         return node;
2899 }
2900
2901 /** Find cover for hashed(nm, nmlen) (or NULL) */
2902 static struct auth_data*
2903 az_nsec3_find_cover(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2904         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2905 {
2906         struct auth_data* node;
2907         uint8_t hname[LDNS_MAX_DOMAINLEN];
2908         size_t hlen = sizeof(hname);
2909         if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2910                 salt, saltlen))
2911                 return NULL;
2912         node = az_nsec3_findnode(z, hname, hlen);
2913         if(node)
2914                 return node;
2915         /* we did not find any, perhaps because the NSEC3 hash is before
2916          * the first hash, we have to find the 'last hash' in the zone */
2917         node = (struct auth_data*)rbtree_last(&z->data);
2918         while(node && (rbnode_type*)node != RBTREE_NULL &&
2919                 !az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2920                 node = (struct auth_data*)rbtree_previous(&node->node);
2921         }
2922         if((rbnode_type*)node == RBTREE_NULL)
2923                 node = NULL;
2924         return node;
2925 }
2926
2927 /** Find exact match for hashed(nm, nmlen) NSEC3 record or NULL */
2928 static struct auth_data*
2929 az_nsec3_find_exact(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2930         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2931 {
2932         struct auth_data* node;
2933         uint8_t hname[LDNS_MAX_DOMAINLEN];
2934         size_t hlen = sizeof(hname);
2935         if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2936                 salt, saltlen))
2937                 return NULL;
2938         node = az_find_name(z, hname, hlen);
2939         if(az_domain_rrset(node, LDNS_RR_TYPE_NSEC3))
2940                 return node;
2941         return NULL;
2942 }
2943
2944 /** Return nextcloser name (as a ref into the qname).  This is one label
2945  * more than the cenm (cename must be a suffix of qname) */
2946 static void
2947 az_nsec3_get_nextcloser(uint8_t* cenm, uint8_t* qname, size_t qname_len,
2948         uint8_t** nx, size_t* nxlen)
2949 {
2950         int celabs = dname_count_labels(cenm);
2951         int qlabs = dname_count_labels(qname);
2952         int strip = qlabs - celabs -1;
2953         log_assert(dname_strict_subdomain(qname, qlabs, cenm, celabs));
2954         *nx = qname;
2955         *nxlen = qname_len;
2956         if(strip>0)
2957                 dname_remove_labels(nx, nxlen, strip);
2958 }
2959
2960 /** Find the closest encloser that has exact NSEC3.
2961  * updated cenm to the new name. If it went up no-exact-ce is true. */
2962 static struct auth_data*
2963 az_nsec3_find_ce(struct auth_zone* z, uint8_t** cenm, size_t* cenmlen,
2964         int* no_exact_ce, int algo, size_t iter, uint8_t* salt, size_t saltlen)
2965 {
2966         struct auth_data* node;
2967         while((node = az_nsec3_find_exact(z, *cenm, *cenmlen,
2968                 algo, iter, salt, saltlen)) == NULL) {
2969                 if(*cenmlen == z->namelen) {
2970                         /* next step up would take us out of the zone. fail */
2971                         return NULL;
2972                 }
2973                 *no_exact_ce = 1;
2974                 dname_remove_label(cenm, cenmlen);
2975         }
2976         return node;
2977 }
2978
2979 /* Insert NSEC3 record in authority section, if NULL does nothing */
2980 static int
2981 az_nsec3_insert(struct auth_zone* z, struct regional* region,
2982         struct dns_msg* msg, struct auth_data* node)
2983 {
2984         struct auth_rrset* nsec3;
2985         if(!node) return 1; /* no node, skip this */
2986         nsec3 = az_domain_rrset(node, LDNS_RR_TYPE_NSEC3);
2987         if(!nsec3) return 1; /* if no nsec3 RR, skip it */
2988         if(!msg_add_rrset_ns(z, region, msg, node, nsec3)) return 0;
2989         return 1;
2990 }
2991
2992 /** add NSEC3 records to the zone for the nsec3 proof.
2993  * Specify with the flags with parts of the proof are required.
2994  * the ce is the exact matching name (for notype) but also delegation points.
2995  * qname is the one where the nextcloser name can be derived from.
2996  * If NSEC3 is not properly there (in the zone) nothing is added.
2997  * always enabled: include nsec3 proving about the Closest Encloser.
2998  *      that is an exact match that should exist for it.
2999  *      If that does not exist, a higher exact match + nxproof is enabled
3000  *      (for some sort of opt-out empty nonterminal cases).
3001  * nodataproof: search for exact match and include that instead.
3002  * ceproof: include ce proof NSEC3 (omitted for wildcard replies).
3003  * nxproof: include denial of the qname.
3004  * wcproof: include denial of wildcard (wildcard.ce).
3005  */
3006 static int
3007 az_add_nsec3_proof(struct auth_zone* z, struct regional* region,
3008         struct dns_msg* msg, uint8_t* cenm, size_t cenmlen, uint8_t* qname,
3009         size_t qname_len, int nodataproof, int ceproof, int nxproof,
3010         int wcproof)
3011 {
3012         int algo;
3013         size_t iter, saltlen;
3014         uint8_t* salt;
3015         int no_exact_ce = 0;
3016         struct auth_data* node;
3017
3018         /* find parameters of nsec3 proof */
3019         if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen))
3020                 return 1; /* no nsec3 */
3021         if(nodataproof) {
3022                 /* see if the node has a hash of itself for the nodata
3023                  * proof nsec3, this has to be an exact match nsec3. */
3024                 struct auth_data* match;
3025                 match = az_nsec3_find_exact(z, qname, qname_len, algo,
3026                         iter, salt, saltlen);
3027                 if(match) {
3028                         if(!az_nsec3_insert(z, region, msg, match))
3029                                 return 0;
3030                         /* only nodata NSEC3 needed, no CE or others. */
3031                         return 1;
3032                 }
3033         }
3034         /* find ce that has an NSEC3 */
3035         if(ceproof) {
3036                 node = az_nsec3_find_ce(z, &cenm, &cenmlen, &no_exact_ce,
3037                         algo, iter, salt, saltlen);
3038                 if(no_exact_ce) nxproof = 1;
3039                 if(!az_nsec3_insert(z, region, msg, node))
3040                         return 0;
3041         }
3042
3043         if(nxproof) {
3044                 uint8_t* nx;
3045                 size_t nxlen;
3046                 /* create nextcloser domain name */
3047                 az_nsec3_get_nextcloser(cenm, qname, qname_len, &nx, &nxlen);
3048                 /* find nsec3 that matches or covers it */
3049                 node = az_nsec3_find_cover(z, nx, nxlen, algo, iter, salt,
3050                         saltlen);
3051                 if(!az_nsec3_insert(z, region, msg, node))
3052                         return 0;
3053         }
3054         if(wcproof) {
3055                 /* create wildcard name *.ce */
3056                 uint8_t wc[LDNS_MAX_DOMAINLEN];
3057                 size_t wclen;
3058                 if(cenmlen+2 > sizeof(wc))
3059                         return 0; /* result would be too long */
3060                 wc[0] = 1; /* length of wildcard label */
3061                 wc[1] = (uint8_t)'*'; /* wildcard label */
3062                 memmove(wc+2, cenm, cenmlen);
3063                 wclen = cenmlen+2;
3064                 /* find nsec3 that matches or covers it */
3065                 node = az_nsec3_find_cover(z, wc, wclen, algo, iter, salt,
3066                         saltlen);
3067                 if(!az_nsec3_insert(z, region, msg, node))
3068                         return 0;
3069         }
3070         return 1;
3071 }
3072
3073 /** generate answer for positive answer */
3074 static int
3075 az_generate_positive_answer(struct auth_zone* z, struct regional* region,
3076         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
3077 {
3078         if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3079         /* see if we want additional rrs */
3080         if(rrset->type == LDNS_RR_TYPE_MX) {
3081                 if(!az_add_additionals_from(z, region, msg, rrset, 2))
3082                         return 0;
3083         } else if(rrset->type == LDNS_RR_TYPE_SRV) {
3084                 if(!az_add_additionals_from(z, region, msg, rrset, 6))
3085                         return 0;
3086         } else if(rrset->type == LDNS_RR_TYPE_NS) {
3087                 if(!az_add_additionals_from(z, region, msg, rrset, 0))
3088                         return 0;
3089         }
3090         return 1;
3091 }
3092
3093 /** generate answer for type ANY answer */
3094 static int
3095 az_generate_any_answer(struct auth_zone* z, struct regional* region,
3096         struct dns_msg* msg, struct auth_data* node)
3097 {
3098         struct auth_rrset* rrset;
3099         int added = 0;
3100         /* add a couple (at least one) RRs */
3101         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_SOA)) != NULL) {
3102                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3103                 added++;
3104         }
3105         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_MX)) != NULL) {
3106                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3107                 added++;
3108         }
3109         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_A)) != NULL) {
3110                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3111                 added++;
3112         }
3113         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_AAAA)) != NULL) {
3114                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3115                 added++;
3116         }
3117         if(added == 0 && node && node->rrsets) {
3118                 if(!msg_add_rrset_an(z, region, msg, node,
3119                         node->rrsets)) return 0;
3120         }
3121         return 1;
3122 }
3123
3124 /** follow cname chain and add more data to the answer section */
3125 static int
3126 follow_cname_chain(struct auth_zone* z, uint16_t qtype,
3127         struct regional* region, struct dns_msg* msg,
3128         struct packed_rrset_data* d)
3129 {
3130         int maxchain = 0;
3131         /* see if we can add the target of the CNAME into the answer */
3132         while(maxchain++ < MAX_CNAME_CHAIN) {
3133                 struct auth_data* node;
3134                 struct auth_rrset* rrset;
3135                 size_t clen;
3136                 /* d has cname rdata */
3137                 if(d->count == 0) break; /* no CNAME */
3138                 if(d->rr_len[0] < 2+1) break; /* too small */
3139                 if((clen=dname_valid(d->rr_data[0]+2, d->rr_len[0]-2))==0)
3140                         break; /* malformed */
3141                 if(!dname_subdomain_c(d->rr_data[0]+2, z->name))
3142                         break; /* target out of zone */
3143                 if((node = az_find_name(z, d->rr_data[0]+2, clen))==NULL)
3144                         break; /* no such target name */
3145                 if((rrset=az_domain_rrset(node, qtype))!=NULL) {
3146                         /* done we found the target */
3147                         if(!msg_add_rrset_an(z, region, msg, node, rrset))
3148                                 return 0;
3149                         break;
3150                 }
3151                 if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME))==NULL)
3152                         break; /* no further CNAME chain, notype */
3153                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3154                 d = rrset->data;
3155         }
3156         return 1;
3157 }
3158
3159 /** generate answer for cname answer */
3160 static int
3161 az_generate_cname_answer(struct auth_zone* z, struct query_info* qinfo,
3162         struct regional* region, struct dns_msg* msg,
3163         struct auth_data* node, struct auth_rrset* rrset)
3164 {
3165         if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3166         if(!rrset) return 1;
3167         if(!follow_cname_chain(z, qinfo->qtype, region, msg, rrset->data))
3168                 return 0;
3169         return 1;
3170 }
3171
3172 /** generate answer for notype answer */
3173 static int
3174 az_generate_notype_answer(struct auth_zone* z, struct regional* region,
3175         struct dns_msg* msg, struct auth_data* node)
3176 {
3177         struct auth_rrset* rrset;
3178         if(!az_add_negative_soa(z, region, msg)) return 0;
3179         /* DNSSEC denial NSEC */
3180         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_NSEC))!=NULL) {
3181                 if(!msg_add_rrset_ns(z, region, msg, node, rrset)) return 0;
3182         } else if(node) {
3183                 /* DNSSEC denial NSEC3 */
3184                 if(!az_add_nsec3_proof(z, region, msg, node->name,
3185                         node->namelen, msg->qinfo.qname,
3186                         msg->qinfo.qname_len, 1, 1, 0, 0))
3187                         return 0;
3188         }
3189         return 1;
3190 }
3191
3192 /** generate answer for referral answer */
3193 static int
3194 az_generate_referral_answer(struct auth_zone* z, struct regional* region,
3195         struct dns_msg* msg, struct auth_data* ce, struct auth_rrset* rrset)
3196 {
3197         struct auth_rrset* ds, *nsec;
3198         /* turn off AA flag, referral is nonAA because it leaves the zone */
3199         log_assert(ce);
3200         msg->rep->flags &= ~BIT_AA;
3201         if(!msg_add_rrset_ns(z, region, msg, ce, rrset)) return 0;
3202         /* add DS or deny it */
3203         if((ds=az_domain_rrset(ce, LDNS_RR_TYPE_DS))!=NULL) {
3204                 if(!msg_add_rrset_ns(z, region, msg, ce, ds)) return 0;
3205         } else {
3206                 /* deny the DS */
3207                 if((nsec=az_domain_rrset(ce, LDNS_RR_TYPE_NSEC))!=NULL) {
3208                         if(!msg_add_rrset_ns(z, region, msg, ce, nsec))
3209                                 return 0;
3210                 } else {
3211                         if(!az_add_nsec3_proof(z, region, msg, ce->name,
3212                                 ce->namelen, msg->qinfo.qname,
3213                                 msg->qinfo.qname_len, 1, 1, 0, 0))
3214                                 return 0;
3215                 }
3216         }
3217         /* add additional rrs for type NS */
3218         if(!az_add_additionals_from(z, region, msg, rrset, 0)) return 0;
3219         return 1;
3220 }
3221
3222 /** generate answer for DNAME answer */
3223 static int
3224 az_generate_dname_answer(struct auth_zone* z, struct query_info* qinfo,
3225         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3226         struct auth_rrset* rrset)
3227 {
3228         log_assert(ce);
3229         /* add the DNAME and then a CNAME */
3230         if(!msg_add_rrset_an(z, region, msg, ce, rrset)) return 0;
3231         if(!add_synth_cname(z, qinfo->qname, qinfo->qname_len, region,
3232                 msg, ce, rrset)) return 0;
3233         if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_YXDOMAIN)
3234                 return 1;
3235         if(msg->rep->rrset_count == 0 ||
3236                 !msg->rep->rrsets[msg->rep->rrset_count-1])
3237                 return 0;
3238         if(!follow_cname_chain(z, qinfo->qtype, region, msg,
3239                 (struct packed_rrset_data*)msg->rep->rrsets[
3240                 msg->rep->rrset_count-1]->entry.data))
3241                 return 0;
3242         return 1;
3243 }
3244
3245 /** generate answer for wildcard answer */
3246 static int
3247 az_generate_wildcard_answer(struct auth_zone* z, struct query_info* qinfo,
3248         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3249         struct auth_data* wildcard, struct auth_data* node)
3250 {
3251         struct auth_rrset* rrset, *nsec;
3252         int insert_ce = 0;
3253         if((rrset=az_domain_rrset(wildcard, qinfo->qtype)) != NULL) {
3254                 /* wildcard has type, add it */
3255                 if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
3256                         return 0;
3257                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3258                         msg->qinfo.qname_len, 1);
3259         } else if((rrset=az_domain_rrset(wildcard, LDNS_RR_TYPE_CNAME))!=NULL) {
3260                 /* wildcard has cname instead, do that */
3261                 if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
3262                         return 0;
3263                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3264                         msg->qinfo.qname_len, 1);
3265                 if(!follow_cname_chain(z, qinfo->qtype, region, msg,
3266                         rrset->data))
3267                         return 0;
3268         } else if(qinfo->qtype == LDNS_RR_TYPE_ANY && wildcard->rrsets) {
3269                 /* add ANY rrsets from wildcard node */
3270                 if(!az_generate_any_answer(z, region, msg, wildcard))
3271                         return 0;
3272                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3273                         msg->qinfo.qname_len, 1);
3274         } else {
3275                 /* wildcard has nodata, notype answer */
3276                 /* call other notype routine for dnssec notype denials */
3277                 if(!az_generate_notype_answer(z, region, msg, wildcard))
3278                         return 0;
3279                 /* because the notype, there is no positive data with an
3280                  * RRSIG that indicates the wildcard position.  Thus the
3281                  * wildcard qname denial needs to have a CE nsec3. */
3282                 insert_ce = 1;
3283         }
3284
3285         /* ce and node for dnssec denial of wildcard original name */
3286         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3287                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3288         } else if(ce) {
3289                 uint8_t* wildup = wildcard->name;
3290                 size_t wilduplen= wildcard->namelen;
3291                 dname_remove_label(&wildup, &wilduplen);
3292                 if(!az_add_nsec3_proof(z, region, msg, wildup,
3293                         wilduplen, msg->qinfo.qname,
3294                         msg->qinfo.qname_len, 0, insert_ce, 1, 0))
3295                         return 0;
3296         }
3297
3298         /* fixup name of wildcard from *.zone to qname, use already allocated
3299          * pointer to msg qname */
3300         az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3301                 msg->qinfo.qname_len, 0);
3302         return 1;
3303 }
3304
3305 /** generate answer for nxdomain answer */
3306 static int
3307 az_generate_nxdomain_answer(struct auth_zone* z, struct regional* region,
3308         struct dns_msg* msg, struct auth_data* ce, struct auth_data* node)
3309 {
3310         struct auth_rrset* nsec;
3311         msg->rep->flags |= LDNS_RCODE_NXDOMAIN;
3312         if(!az_add_negative_soa(z, region, msg)) return 0;
3313         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3314                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3315                 if(ce && !az_nsec_wildcard_denial(z, region, msg, ce->name,
3316                         ce->namelen)) return 0;
3317         } else if(ce) {
3318                 if(!az_add_nsec3_proof(z, region, msg, ce->name,
3319                         ce->namelen, msg->qinfo.qname,
3320                         msg->qinfo.qname_len, 0, 1, 1, 1))
3321                         return 0;
3322         }
3323         return 1;
3324 }
3325
3326 /** Create answers when an exact match exists for the domain name */
3327 static int
3328 az_generate_answer_with_node(struct auth_zone* z, struct query_info* qinfo,
3329         struct regional* region, struct dns_msg* msg, struct auth_data* node)
3330 {
3331         struct auth_rrset* rrset;
3332         /* positive answer, rrset we are looking for exists */
3333         if((rrset=az_domain_rrset(node, qinfo->qtype)) != NULL) {
3334                 return az_generate_positive_answer(z, region, msg, node, rrset);
3335         }
3336         /* CNAME? */
3337         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME)) != NULL) {
3338                 return az_generate_cname_answer(z, qinfo, region, msg,
3339                         node, rrset);
3340         }
3341         /* type ANY ? */
3342         if(qinfo->qtype == LDNS_RR_TYPE_ANY) {
3343                 return az_generate_any_answer(z, region, msg, node);
3344         }
3345         /* NOERROR/NODATA (no such type at domain name) */
3346         return az_generate_notype_answer(z, region, msg, node);
3347 }
3348
3349 /** Generate answer without an existing-node that we can use.
3350  * So it'll be a referral, DNAME or nxdomain */
3351 static int
3352 az_generate_answer_nonexistnode(struct auth_zone* z, struct query_info* qinfo,
3353         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3354         struct auth_rrset* rrset, struct auth_data* node)
3355 {
3356         struct auth_data* wildcard;
3357
3358         /* we do not have an exact matching name (that exists) */
3359         /* see if we have a NS or DNAME in the ce */
3360         if(ce && rrset && rrset->type == LDNS_RR_TYPE_NS) {
3361                 return az_generate_referral_answer(z, region, msg, ce, rrset);
3362         }
3363         if(ce && rrset && rrset->type == LDNS_RR_TYPE_DNAME) {
3364                 return az_generate_dname_answer(z, qinfo, region, msg, ce,
3365                         rrset);
3366         }
3367         /* if there is an empty nonterminal, wildcard and nxdomain don't
3368          * happen, it is a notype answer */
3369         if(az_empty_nonterminal(z, qinfo, node)) {
3370                 return az_generate_notype_answer(z, region, msg, node);
3371         }
3372         /* see if we have a wildcard under the ce */
3373         if((wildcard=az_find_wildcard(z, qinfo, ce)) != NULL) {
3374                 return az_generate_wildcard_answer(z, qinfo, region, msg,
3375                         ce, wildcard, node);
3376         }
3377         /* generate nxdomain answer */
3378         return az_generate_nxdomain_answer(z, region, msg, ce, node);
3379 }
3380
3381 /** Lookup answer in a zone. */
3382 static int
3383 auth_zone_generate_answer(struct auth_zone* z, struct query_info* qinfo,
3384         struct regional* region, struct dns_msg** msg, int* fallback)
3385 {
3386         struct auth_data* node, *ce;
3387         struct auth_rrset* rrset;
3388         int node_exact, node_exists;
3389         /* does the zone want fallback in case of failure? */
3390         *fallback = z->fallback_enabled;
3391         if(!(*msg=msg_create(region, qinfo))) return 0;
3392
3393         /* lookup if there is a matching domain name for the query */
3394         az_find_domain(z, qinfo, &node_exact, &node);
3395
3396         /* see if node exists for generating answers from (i.e. not glue and
3397          * obscured by NS or DNAME or NSEC3-only), and also return the
3398          * closest-encloser from that, closest node that should be used
3399          * to generate answers from that is above the query */
3400         node_exists = az_find_ce(z, qinfo, node, node_exact, &ce, &rrset);
3401
3402         if(verbosity >= VERB_ALGO) {
3403                 char zname[256], qname[256], nname[256], cename[256],
3404                         tpstr[32], rrstr[32];
3405                 sldns_wire2str_dname_buf(qinfo->qname, qinfo->qname_len, qname,
3406                         sizeof(qname));
3407                 sldns_wire2str_type_buf(qinfo->qtype, tpstr, sizeof(tpstr));
3408                 sldns_wire2str_dname_buf(z->name, z->namelen, zname,
3409                         sizeof(zname));
3410                 if(node)
3411                         sldns_wire2str_dname_buf(node->name, node->namelen,
3412                                 nname, sizeof(nname));
3413                 else    snprintf(nname, sizeof(nname), "NULL");
3414                 if(ce)
3415                         sldns_wire2str_dname_buf(ce->name, ce->namelen,
3416                                 cename, sizeof(cename));
3417                 else    snprintf(cename, sizeof(cename), "NULL");
3418                 if(rrset) sldns_wire2str_type_buf(rrset->type, rrstr,
3419                         sizeof(rrstr));
3420                 else    snprintf(rrstr, sizeof(rrstr), "NULL");
3421                 log_info("auth_zone %s query %s %s, domain %s %s %s, "
3422                         "ce %s, rrset %s", zname, qname, tpstr, nname,
3423                         (node_exact?"exact":"notexact"),
3424                         (node_exists?"exist":"notexist"), cename, rrstr);
3425         }
3426
3427         if(node_exists) {
3428                 /* the node is fine, generate answer from node */
3429                 return az_generate_answer_with_node(z, qinfo, region, *msg,
3430                         node);
3431         }
3432         return az_generate_answer_nonexistnode(z, qinfo, region, *msg,
3433                 ce, rrset, node);
3434 }
3435
3436 int auth_zones_lookup(struct auth_zones* az, struct query_info* qinfo,
3437         struct regional* region, struct dns_msg** msg, int* fallback,
3438         uint8_t* dp_nm, size_t dp_nmlen)
3439 {
3440         int r;
3441         struct auth_zone* z;
3442         /* find the zone that should contain the answer. */
3443         lock_rw_rdlock(&az->lock);
3444         z = auth_zone_find(az, dp_nm, dp_nmlen, qinfo->qclass);
3445         if(!z) {
3446                 lock_rw_unlock(&az->lock);
3447                 /* no auth zone, fallback to internet */
3448                 *fallback = 1;
3449                 return 0;
3450         }
3451         lock_rw_rdlock(&z->lock);
3452         lock_rw_unlock(&az->lock);
3453
3454         /* if not for upstream queries, fallback */
3455         if(!z->for_upstream) {
3456                 lock_rw_unlock(&z->lock);
3457                 *fallback = 1;
3458                 return 0;
3459         }
3460         if(z->zone_expired) {
3461                 *fallback = z->fallback_enabled;
3462                 lock_rw_unlock(&z->lock);
3463                 return 0;
3464         }
3465         /* see what answer that zone would generate */
3466         r = auth_zone_generate_answer(z, qinfo, region, msg, fallback);
3467         lock_rw_unlock(&z->lock);
3468         return r;
3469 }
3470
3471 /** encode auth answer */
3472 static void
3473 auth_answer_encode(struct query_info* qinfo, struct module_env* env,
3474         struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3475         struct regional* temp, struct dns_msg* msg)
3476 {
3477         uint16_t udpsize;
3478         udpsize = edns->udp_size;
3479         edns->edns_version = EDNS_ADVERTISED_VERSION;
3480         edns->udp_size = EDNS_ADVERTISED_SIZE;
3481         edns->ext_rcode = 0;
3482         edns->bits &= EDNS_DO;
3483
3484         if(!inplace_cb_reply_local_call(env, qinfo, NULL, msg->rep,
3485                 (int)FLAGS_GET_RCODE(msg->rep->flags), edns, repinfo, temp, env->now_tv)
3486                 || !reply_info_answer_encode(qinfo, msg->rep,
3487                 *(uint16_t*)sldns_buffer_begin(buf),
3488                 sldns_buffer_read_u16_at(buf, 2),
3489                 buf, 0, 0, temp, udpsize, edns,
3490                 (int)(edns->bits&EDNS_DO), 0)) {
3491                 error_encode(buf, (LDNS_RCODE_SERVFAIL|BIT_AA), qinfo,
3492                         *(uint16_t*)sldns_buffer_begin(buf),
3493                         sldns_buffer_read_u16_at(buf, 2), edns);
3494         }
3495 }
3496
3497 /** encode auth error answer */
3498 static void
3499 auth_error_encode(struct query_info* qinfo, struct module_env* env,
3500         struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3501         struct regional* temp, int rcode)
3502 {
3503         edns->edns_version = EDNS_ADVERTISED_VERSION;
3504         edns->udp_size = EDNS_ADVERTISED_SIZE;
3505         edns->ext_rcode = 0;
3506         edns->bits &= EDNS_DO;
3507
3508         if(!inplace_cb_reply_local_call(env, qinfo, NULL, NULL,
3509                 rcode, edns, repinfo, temp, env->now_tv))
3510                 edns->opt_list = NULL;
3511         error_encode(buf, rcode|BIT_AA, qinfo,
3512                 *(uint16_t*)sldns_buffer_begin(buf),
3513                 sldns_buffer_read_u16_at(buf, 2), edns);
3514 }
3515
3516 int auth_zones_answer(struct auth_zones* az, struct module_env* env,
3517         struct query_info* qinfo, struct edns_data* edns,
3518         struct comm_reply* repinfo, struct sldns_buffer* buf, struct regional* temp)
3519 {
3520         struct dns_msg* msg = NULL;
3521         struct auth_zone* z;
3522         int r;
3523         int fallback = 0;
3524
3525         lock_rw_rdlock(&az->lock);
3526         if(!az->have_downstream) {
3527                 /* no downstream auth zones */
3528                 lock_rw_unlock(&az->lock);
3529                 return 0;
3530         }
3531         if(qinfo->qtype == LDNS_RR_TYPE_DS) {
3532                 uint8_t* delname = qinfo->qname;
3533                 size_t delnamelen = qinfo->qname_len;
3534                 dname_remove_label(&delname, &delnamelen);
3535                 z = auth_zones_find_zone(az, delname, delnamelen,
3536                         qinfo->qclass);
3537         } else {
3538                 z = auth_zones_find_zone(az, qinfo->qname, qinfo->qname_len,
3539                         qinfo->qclass);
3540         }
3541         if(!z) {
3542                 /* no zone above it */
3543                 lock_rw_unlock(&az->lock);
3544                 return 0;
3545         }
3546         lock_rw_rdlock(&z->lock);
3547         lock_rw_unlock(&az->lock);
3548         if(!z->for_downstream) {
3549                 lock_rw_unlock(&z->lock);
3550                 return 0;
3551         }
3552         if(z->zone_expired) {
3553                 if(z->fallback_enabled) {
3554                         lock_rw_unlock(&z->lock);
3555                         return 0;
3556                 }
3557                 lock_rw_unlock(&z->lock);
3558                 lock_rw_wrlock(&az->lock);
3559                 az->num_query_down++;
3560                 lock_rw_unlock(&az->lock);
3561                 auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3562                         LDNS_RCODE_SERVFAIL);
3563                 return 1;
3564         }
3565
3566         /* answer it from zone z */
3567         r = auth_zone_generate_answer(z, qinfo, temp, &msg, &fallback);
3568         lock_rw_unlock(&z->lock);
3569         if(!r && fallback) {
3570                 /* fallback to regular answering (recursive) */
3571                 return 0;
3572         }
3573         lock_rw_wrlock(&az->lock);
3574         az->num_query_down++;
3575         lock_rw_unlock(&az->lock);
3576
3577         /* encode answer */
3578         if(!r)
3579                 auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3580                         LDNS_RCODE_SERVFAIL);
3581         else    auth_answer_encode(qinfo, env, edns, repinfo, buf, temp, msg);
3582
3583         return 1;
3584 }
3585
3586 int auth_zones_can_fallback(struct auth_zones* az, uint8_t* nm, size_t nmlen,
3587         uint16_t dclass)
3588 {
3589         int r;
3590         struct auth_zone* z;
3591         lock_rw_rdlock(&az->lock);
3592         z = auth_zone_find(az, nm, nmlen, dclass);
3593         if(!z) {
3594                 lock_rw_unlock(&az->lock);
3595                 /* no such auth zone, fallback */
3596                 return 1;
3597         }
3598         lock_rw_rdlock(&z->lock);
3599         lock_rw_unlock(&az->lock);
3600         r = z->fallback_enabled || (!z->for_upstream);
3601         lock_rw_unlock(&z->lock);
3602         return r;
3603 }
3604
3605 int
3606 auth_zone_parse_notify_serial(sldns_buffer* pkt, uint32_t *serial)
3607 {
3608         struct query_info q;
3609         uint16_t rdlen;
3610         memset(&q, 0, sizeof(q));
3611         sldns_buffer_set_position(pkt, 0);
3612         if(!query_info_parse(&q, pkt)) return 0;
3613         if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0) return 0;
3614         /* skip name of RR in answer section */
3615         if(sldns_buffer_remaining(pkt) < 1) return 0;
3616         if(pkt_dname_len(pkt) == 0) return 0;
3617         /* check type */
3618         if(sldns_buffer_remaining(pkt) < 10 /* type,class,ttl,rdatalen*/)
3619                 return 0;
3620         if(sldns_buffer_read_u16(pkt) != LDNS_RR_TYPE_SOA) return 0;
3621         sldns_buffer_skip(pkt, 2); /* class */
3622         sldns_buffer_skip(pkt, 4); /* ttl */
3623         rdlen = sldns_buffer_read_u16(pkt); /* rdatalen */
3624         if(sldns_buffer_remaining(pkt) < rdlen) return 0;
3625         if(rdlen < 22) return 0; /* bad soa length */
3626         sldns_buffer_skip(pkt, (ssize_t)(rdlen-20));
3627         *serial = sldns_buffer_read_u32(pkt);
3628         /* return true when has serial in answer section */
3629         return 1;
3630 }
3631
3632 /** see if addr appears in the list */
3633 static int
3634 addr_in_list(struct auth_addr* list, struct sockaddr_storage* addr,
3635         socklen_t addrlen)
3636 {
3637         struct auth_addr* p;
3638         for(p=list; p; p=p->next) {
3639                 if(sockaddr_cmp_addr(addr, addrlen, &p->addr, p->addrlen)==0)
3640                         return 1;
3641         }
3642         return 0;
3643 }
3644
3645 /** check if an address matches a master specification (or one of its
3646  * addresses in the addr list) */
3647 static int
3648 addr_matches_master(struct auth_master* master, struct sockaddr_storage* addr,
3649         socklen_t addrlen, struct auth_master** fromhost)
3650 {
3651         struct sockaddr_storage a;
3652         socklen_t alen = 0;
3653         int net = 0;
3654         if(addr_in_list(master->list, addr, addrlen)) {
3655                 *fromhost = master;
3656                 return 1;
3657         }
3658         /* compare address (but not port number, that is the destination
3659          * port of the master, the port number of the received notify is
3660          * allowed to by any port on that master) */
3661         if(extstrtoaddr(master->host, &a, &alen) &&
3662                 sockaddr_cmp_addr(addr, addrlen, &a, alen)==0) {
3663                 *fromhost = master;
3664                 return 1;
3665         }
3666         /* prefixes, addr/len, like 10.0.0.0/8 */
3667         /* not http and has a / and there is one / */
3668         if(master->allow_notify && !master->http &&
3669                 strchr(master->host, '/') != NULL &&
3670                 strchr(master->host, '/') == strrchr(master->host, '/') &&
3671                 netblockstrtoaddr(master->host, UNBOUND_DNS_PORT, &a, &alen,
3672                 &net) && alen == addrlen) {
3673                 if(addr_in_common(addr, (addr_is_ip6(addr, addrlen)?128:32),
3674                         &a, net, alen) >= net) {
3675                         *fromhost = NULL; /* prefix does not have destination
3676                                 to send the probe or transfer with */
3677                         return 1; /* matches the netblock */
3678                 }
3679         }
3680         return 0;
3681 }
3682
3683 /** check access list for notifies */
3684 static int
3685 az_xfr_allowed_notify(struct auth_xfer* xfr, struct sockaddr_storage* addr,
3686         socklen_t addrlen, struct auth_master** fromhost)
3687 {
3688         struct auth_master* p;
3689         for(p=xfr->allow_notify_list; p; p=p->next) {
3690                 if(addr_matches_master(p, addr, addrlen, fromhost)) {
3691                         return 1;
3692                 }
3693         }
3694         return 0;
3695 }
3696
3697 /** see if the serial means the zone has to be updated, i.e. the serial
3698  * is newer than the zone serial, or we have no zone */
3699 static int
3700 xfr_serial_means_update(struct auth_xfer* xfr, uint32_t serial)
3701 {
3702         if(!xfr->have_zone)
3703                 return 1; /* no zone, anything is better */
3704         if(xfr->zone_expired)
3705                 return 1; /* expired, the sent serial is better than expired
3706                         data */
3707         if(compare_serial(xfr->serial, serial) < 0)
3708                 return 1; /* our serial is smaller than the sent serial,
3709                         the data is newer, fetch it */
3710         return 0;
3711 }
3712
3713 /** note notify serial, updates the notify information in the xfr struct */
3714 static void
3715 xfr_note_notify_serial(struct auth_xfer* xfr, int has_serial, uint32_t serial)
3716 {
3717         if(xfr->notify_received && xfr->notify_has_serial && has_serial) {
3718                 /* see if this serial is newer */
3719                 if(compare_serial(xfr->notify_serial, serial) < 0)
3720                         xfr->notify_serial = serial;
3721         } else if(xfr->notify_received && xfr->notify_has_serial &&
3722                 !has_serial) {
3723                 /* remove serial, we have notify without serial */
3724                 xfr->notify_has_serial = 0;
3725                 xfr->notify_serial = 0;
3726         } else if(xfr->notify_received && !xfr->notify_has_serial) {
3727                 /* we already have notify without serial, keep it
3728                  * that way; no serial check when current operation
3729                  * is done */
3730         } else {
3731                 xfr->notify_received = 1;
3732                 xfr->notify_has_serial = has_serial;
3733                 xfr->notify_serial = serial;
3734         }
3735 }
3736
3737 /** process a notify serial, start new probe or note serial. xfr is locked */
3738 static void
3739 xfr_process_notify(struct auth_xfer* xfr, struct module_env* env,
3740         int has_serial, uint32_t serial, struct auth_master* fromhost)
3741 {
3742         /* if the serial of notify is older than we have, don't fetch
3743          * a zone, we already have it */
3744         if(has_serial && !xfr_serial_means_update(xfr, serial)) {
3745                 lock_basic_unlock(&xfr->lock);
3746                 return;
3747         }
3748         /* start new probe with this addr src, or note serial */
3749         if(!xfr_start_probe(xfr, env, fromhost)) {
3750                 /* not started because already in progress, note the serial */
3751                 xfr_note_notify_serial(xfr, has_serial, serial);
3752                 lock_basic_unlock(&xfr->lock);
3753         }
3754         /* successful end of start_probe unlocked xfr->lock */
3755 }
3756
3757 int auth_zones_notify(struct auth_zones* az, struct module_env* env,
3758         uint8_t* nm, size_t nmlen, uint16_t dclass,
3759         struct sockaddr_storage* addr, socklen_t addrlen, int has_serial,
3760         uint32_t serial, int* refused)
3761 {
3762         struct auth_xfer* xfr;
3763         struct auth_master* fromhost = NULL;
3764         /* see which zone this is */
3765         lock_rw_rdlock(&az->lock);
3766         xfr = auth_xfer_find(az, nm, nmlen, dclass);
3767         if(!xfr) {
3768                 lock_rw_unlock(&az->lock);
3769                 /* no such zone, refuse the notify */
3770                 *refused = 1;
3771                 return 0;
3772         }
3773         lock_basic_lock(&xfr->lock);
3774         lock_rw_unlock(&az->lock);
3775
3776         /* check access list for notifies */
3777         if(!az_xfr_allowed_notify(xfr, addr, addrlen, &fromhost)) {
3778                 lock_basic_unlock(&xfr->lock);
3779                 /* notify not allowed, refuse the notify */
3780                 *refused = 1;
3781                 return 0;
3782         }
3783
3784         /* process the notify */
3785         xfr_process_notify(xfr, env, has_serial, serial, fromhost);
3786         return 1;
3787 }
3788
3789 int auth_zones_startprobesequence(struct auth_zones* az,
3790         struct module_env* env, uint8_t* nm, size_t nmlen, uint16_t dclass)
3791 {
3792         struct auth_xfer* xfr;
3793         lock_rw_rdlock(&az->lock);
3794         xfr = auth_xfer_find(az, nm, nmlen, dclass);
3795         if(!xfr) {
3796                 lock_rw_unlock(&az->lock);
3797                 return 0;
3798         }
3799         lock_basic_lock(&xfr->lock);
3800         lock_rw_unlock(&az->lock);
3801
3802         xfr_process_notify(xfr, env, 0, 0, NULL);
3803         return 1;
3804 }
3805
3806 /** set a zone expired */
3807 static void
3808 auth_xfer_set_expired(struct auth_xfer* xfr, struct module_env* env,
3809         int expired)
3810 {
3811         struct auth_zone* z;
3812
3813         /* expire xfr */
3814         lock_basic_lock(&xfr->lock);
3815         xfr->zone_expired = expired;
3816         lock_basic_unlock(&xfr->lock);
3817
3818         /* find auth_zone */
3819         lock_rw_rdlock(&env->auth_zones->lock);
3820         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
3821                 xfr->dclass);
3822         if(!z) {
3823                 lock_rw_unlock(&env->auth_zones->lock);
3824                 return;
3825         }
3826         lock_rw_wrlock(&z->lock);
3827         lock_rw_unlock(&env->auth_zones->lock);
3828
3829         /* expire auth_zone */
3830         z->zone_expired = expired;
3831         lock_rw_unlock(&z->lock);
3832 }
3833
3834 /** find master (from notify or probe) in list of masters */
3835 static struct auth_master*
3836 find_master_by_host(struct auth_master* list, char* host)
3837 {
3838         struct auth_master* p;
3839         for(p=list; p; p=p->next) {
3840                 if(strcmp(p->host, host) == 0)
3841                         return p;
3842         }
3843         return NULL;
3844 }
3845
3846 /** delete the looked up auth_addrs for all the masters in the list */
3847 static void
3848 xfr_masterlist_free_addrs(struct auth_master* list)
3849 {
3850         struct auth_master* m;
3851         for(m=list; m; m=m->next) {
3852                 if(m->list) {
3853                         auth_free_master_addrs(m->list);
3854                         m->list = NULL;
3855                 }
3856         }
3857 }
3858
3859 /** copy a list of auth_addrs */
3860 static struct auth_addr*
3861 auth_addr_list_copy(struct auth_addr* source)
3862 {
3863         struct auth_addr* list = NULL, *last = NULL;
3864         struct auth_addr* p;
3865         for(p=source; p; p=p->next) {
3866                 struct auth_addr* a = (struct auth_addr*)memdup(p, sizeof(*p));
3867                 if(!a) {
3868                         log_err("malloc failure");
3869                         auth_free_master_addrs(list);
3870                         return NULL;
3871                 }
3872                 a->next = NULL;
3873                 if(last) last->next = a;
3874                 if(!list) list = a;
3875                 last = a;
3876         }
3877         return list;
3878 }
3879
3880 /** copy a master to a new structure, NULL on alloc failure */
3881 static struct auth_master*
3882 auth_master_copy(struct auth_master* o)
3883 {
3884         struct auth_master* m;
3885         if(!o) return NULL;
3886         m = (struct auth_master*)memdup(o, sizeof(*o));
3887         if(!m) {
3888                 log_err("malloc failure");
3889                 return NULL;
3890         }
3891         m->next = NULL;
3892         if(m->host) {
3893                 m->host = strdup(m->host);
3894                 if(!m->host) {
3895                         free(m);
3896                         log_err("malloc failure");
3897                         return NULL;
3898                 }
3899         }
3900         if(m->file) {
3901                 m->file = strdup(m->file);
3902                 if(!m->file) {
3903                         free(m->host);
3904                         free(m);
3905                         log_err("malloc failure");
3906                         return NULL;
3907                 }
3908         }
3909         if(m->list) {
3910                 m->list = auth_addr_list_copy(m->list);
3911                 if(!m->list) {
3912                         free(m->file);
3913                         free(m->host);
3914                         free(m);
3915                         return NULL;
3916                 }
3917         }
3918         return m;
3919 }
3920
3921 /** copy the master addresses from the task_probe lookups to the allow_notify
3922  * list of masters */
3923 static void
3924 probe_copy_masters_for_allow_notify(struct auth_xfer* xfr)
3925 {
3926         struct auth_master* list = NULL, *last = NULL;
3927         struct auth_master* p;
3928         /* build up new list with copies */
3929         for(p = xfr->task_probe->masters; p; p=p->next) {
3930                 struct auth_master* m = auth_master_copy(p);
3931                 if(!m) {
3932                         auth_free_masters(list);
3933                         /* failed because of malloc failure, use old list */
3934                         return;
3935                 }
3936                 m->next = NULL;
3937                 if(last) last->next = m;
3938                 if(!list) list = m;
3939                 last = m;
3940         }
3941         /* success, replace list */
3942         auth_free_masters(xfr->allow_notify_list);
3943         xfr->allow_notify_list = list;
3944 }
3945
3946 /** start the lookups for task_transfer */
3947 static void
3948 xfr_transfer_start_lookups(struct auth_xfer* xfr)
3949 {
3950         /* delete all the looked up addresses in the list */
3951         xfr->task_transfer->scan_addr = NULL;
3952         xfr_masterlist_free_addrs(xfr->task_transfer->masters);
3953
3954         /* start lookup at the first master */
3955         xfr->task_transfer->lookup_target = xfr->task_transfer->masters;
3956         xfr->task_transfer->lookup_aaaa = 0;
3957 }
3958
3959 /** move to the next lookup of hostname for task_transfer */
3960 static void
3961 xfr_transfer_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3962 {
3963         if(!xfr->task_transfer->lookup_target)
3964                 return; /* already at end of list */
3965         if(!xfr->task_transfer->lookup_aaaa && env->cfg->do_ip6) {
3966                 /* move to lookup AAAA */
3967                 xfr->task_transfer->lookup_aaaa = 1;
3968                 return;
3969         }
3970         xfr->task_transfer->lookup_target =
3971                 xfr->task_transfer->lookup_target->next;
3972         xfr->task_transfer->lookup_aaaa = 0;
3973         if(!env->cfg->do_ip4 && xfr->task_transfer->lookup_target!=NULL)
3974                 xfr->task_transfer->lookup_aaaa = 1;
3975 }
3976
3977 /** start the lookups for task_probe */
3978 static void
3979 xfr_probe_start_lookups(struct auth_xfer* xfr)
3980 {
3981         /* delete all the looked up addresses in the list */
3982         xfr->task_probe->scan_addr = NULL;
3983         xfr_masterlist_free_addrs(xfr->task_probe->masters);
3984
3985         /* start lookup at the first master */
3986         xfr->task_probe->lookup_target = xfr->task_probe->masters;
3987         xfr->task_probe->lookup_aaaa = 0;
3988 }
3989
3990 /** move to the next lookup of hostname for task_probe */
3991 static void
3992 xfr_probe_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3993 {
3994         if(!xfr->task_probe->lookup_target)
3995                 return; /* already at end of list */
3996         if(!xfr->task_probe->lookup_aaaa && env->cfg->do_ip6) {
3997                 /* move to lookup AAAA */
3998                 xfr->task_probe->lookup_aaaa = 1;
3999                 return;
4000         }
4001         xfr->task_probe->lookup_target = xfr->task_probe->lookup_target->next;
4002         xfr->task_probe->lookup_aaaa = 0;
4003         if(!env->cfg->do_ip4 && xfr->task_probe->lookup_target!=NULL)
4004                 xfr->task_probe->lookup_aaaa = 1;
4005 }
4006
4007 /** start the iteration of the task_transfer list of masters */
4008 static void
4009 xfr_transfer_start_list(struct auth_xfer* xfr, struct auth_master* spec)
4010 {
4011         if(spec) {
4012                 xfr->task_transfer->scan_specific = find_master_by_host(
4013                         xfr->task_transfer->masters, spec->host);
4014                 if(xfr->task_transfer->scan_specific) {
4015                         xfr->task_transfer->scan_target = NULL;
4016                         xfr->task_transfer->scan_addr = NULL;
4017                         if(xfr->task_transfer->scan_specific->list)
4018                                 xfr->task_transfer->scan_addr =
4019                                         xfr->task_transfer->scan_specific->list;
4020                         return;
4021                 }
4022         }
4023         /* no specific (notified) host to scan */
4024         xfr->task_transfer->scan_specific = NULL;
4025         xfr->task_transfer->scan_addr = NULL;
4026         /* pick up first scan target */
4027         xfr->task_transfer->scan_target = xfr->task_transfer->masters;
4028         if(xfr->task_transfer->scan_target && xfr->task_transfer->
4029                 scan_target->list)
4030                 xfr->task_transfer->scan_addr =
4031                         xfr->task_transfer->scan_target->list;
4032 }
4033
4034 /** start the iteration of the task_probe list of masters */
4035 static void
4036 xfr_probe_start_list(struct auth_xfer* xfr, struct auth_master* spec)
4037 {
4038         if(spec) {
4039                 xfr->task_probe->scan_specific = find_master_by_host(
4040                         xfr->task_probe->masters, spec->host);
4041                 if(xfr->task_probe->scan_specific) {
4042                         xfr->task_probe->scan_target = NULL;
4043                         xfr->task_probe->scan_addr = NULL;
4044                         if(xfr->task_probe->scan_specific->list)
4045                                 xfr->task_probe->scan_addr =
4046                                         xfr->task_probe->scan_specific->list;
4047                         return;
4048                 }
4049         }
4050         /* no specific (notified) host to scan */
4051         xfr->task_probe->scan_specific = NULL;
4052         xfr->task_probe->scan_addr = NULL;
4053         /* pick up first scan target */
4054         xfr->task_probe->scan_target = xfr->task_probe->masters;
4055         if(xfr->task_probe->scan_target && xfr->task_probe->scan_target->list)
4056                 xfr->task_probe->scan_addr =
4057                         xfr->task_probe->scan_target->list;
4058 }
4059
4060 /** pick up the master that is being scanned right now, task_transfer */
4061 static struct auth_master*
4062 xfr_transfer_current_master(struct auth_xfer* xfr)
4063 {
4064         if(xfr->task_transfer->scan_specific)
4065                 return xfr->task_transfer->scan_specific;
4066         return xfr->task_transfer->scan_target;
4067 }
4068
4069 /** pick up the master that is being scanned right now, task_probe */
4070 static struct auth_master*
4071 xfr_probe_current_master(struct auth_xfer* xfr)
4072 {
4073         if(xfr->task_probe->scan_specific)
4074                 return xfr->task_probe->scan_specific;
4075         return xfr->task_probe->scan_target;
4076 }
4077
4078 /** true if at end of list, task_transfer */
4079 static int
4080 xfr_transfer_end_of_list(struct auth_xfer* xfr)
4081 {
4082         return !xfr->task_transfer->scan_specific &&
4083                 !xfr->task_transfer->scan_target;
4084 }
4085
4086 /** true if at end of list, task_probe */
4087 static int
4088 xfr_probe_end_of_list(struct auth_xfer* xfr)
4089 {
4090         return !xfr->task_probe->scan_specific && !xfr->task_probe->scan_target;
4091 }
4092
4093 /** move to next master in list, task_transfer */
4094 static void
4095 xfr_transfer_nextmaster(struct auth_xfer* xfr)
4096 {
4097         if(!xfr->task_transfer->scan_specific &&
4098                 !xfr->task_transfer->scan_target)
4099                 return;
4100         if(xfr->task_transfer->scan_addr) {
4101                 xfr->task_transfer->scan_addr =
4102                         xfr->task_transfer->scan_addr->next;
4103                 if(xfr->task_transfer->scan_addr)
4104                         return;
4105         }
4106         if(xfr->task_transfer->scan_specific) {
4107                 xfr->task_transfer->scan_specific = NULL;
4108                 xfr->task_transfer->scan_target = xfr->task_transfer->masters;
4109                 if(xfr->task_transfer->scan_target && xfr->task_transfer->
4110                         scan_target->list)
4111                         xfr->task_transfer->scan_addr =
4112                                 xfr->task_transfer->scan_target->list;
4113                 return;
4114         }
4115         if(!xfr->task_transfer->scan_target)
4116                 return;
4117         xfr->task_transfer->scan_target = xfr->task_transfer->scan_target->next;
4118         if(xfr->task_transfer->scan_target && xfr->task_transfer->
4119                 scan_target->list)
4120                 xfr->task_transfer->scan_addr =
4121                         xfr->task_transfer->scan_target->list;
4122         return;
4123 }
4124
4125 /** move to next master in list, task_probe */
4126 static void
4127 xfr_probe_nextmaster(struct auth_xfer* xfr)
4128 {
4129         if(!xfr->task_probe->scan_specific && !xfr->task_probe->scan_target)
4130                 return;
4131         if(xfr->task_probe->scan_addr) {
4132                 xfr->task_probe->scan_addr = xfr->task_probe->scan_addr->next;
4133                 if(xfr->task_probe->scan_addr)
4134                         return;
4135         }
4136         if(xfr->task_probe->scan_specific) {
4137                 xfr->task_probe->scan_specific = NULL;
4138                 xfr->task_probe->scan_target = xfr->task_probe->masters;
4139                 if(xfr->task_probe->scan_target && xfr->task_probe->
4140                         scan_target->list)
4141                         xfr->task_probe->scan_addr =
4142                                 xfr->task_probe->scan_target->list;
4143                 return;
4144         }
4145         if(!xfr->task_probe->scan_target)
4146                 return;
4147         xfr->task_probe->scan_target = xfr->task_probe->scan_target->next;
4148         if(xfr->task_probe->scan_target && xfr->task_probe->
4149                 scan_target->list)
4150                 xfr->task_probe->scan_addr =
4151                         xfr->task_probe->scan_target->list;
4152         return;
4153 }
4154
4155 /** create SOA probe packet for xfr */
4156 static void
4157 xfr_create_soa_probe_packet(struct auth_xfer* xfr, sldns_buffer* buf,
4158         uint16_t id)
4159 {
4160         struct query_info qinfo;
4161
4162         memset(&qinfo, 0, sizeof(qinfo));
4163         qinfo.qname = xfr->name;
4164         qinfo.qname_len = xfr->namelen;
4165         qinfo.qtype = LDNS_RR_TYPE_SOA;
4166         qinfo.qclass = xfr->dclass;
4167         qinfo_query_encode(buf, &qinfo);
4168         sldns_buffer_write_u16_at(buf, 0, id);
4169 }
4170
4171 /** create IXFR/AXFR packet for xfr */
4172 static void
4173 xfr_create_ixfr_packet(struct auth_xfer* xfr, sldns_buffer* buf, uint16_t id,
4174         struct auth_master* master)
4175 {
4176         struct query_info qinfo;
4177         uint32_t serial;
4178         int have_zone;
4179         have_zone = xfr->have_zone;
4180         serial = xfr->serial;
4181
4182         memset(&qinfo, 0, sizeof(qinfo));
4183         qinfo.qname = xfr->name;
4184         qinfo.qname_len = xfr->namelen;
4185         xfr->task_transfer->got_xfr_serial = 0;
4186         xfr->task_transfer->rr_scan_num = 0;
4187         xfr->task_transfer->incoming_xfr_serial = 0;
4188         xfr->task_transfer->on_ixfr_is_axfr = 0;
4189         xfr->task_transfer->on_ixfr = 1;
4190         qinfo.qtype = LDNS_RR_TYPE_IXFR;
4191         if(!have_zone || xfr->task_transfer->ixfr_fail || !master->ixfr) {
4192                 qinfo.qtype = LDNS_RR_TYPE_AXFR;
4193                 xfr->task_transfer->ixfr_fail = 0;
4194                 xfr->task_transfer->on_ixfr = 0;
4195         }
4196
4197         qinfo.qclass = xfr->dclass;
4198         qinfo_query_encode(buf, &qinfo);
4199         sldns_buffer_write_u16_at(buf, 0, id);
4200
4201         /* append serial for IXFR */
4202         if(qinfo.qtype == LDNS_RR_TYPE_IXFR) {
4203                 size_t end = sldns_buffer_limit(buf);
4204                 sldns_buffer_clear(buf);
4205                 sldns_buffer_set_position(buf, end);
4206                 /* auth section count 1 */
4207                 sldns_buffer_write_u16_at(buf, LDNS_NSCOUNT_OFF, 1);
4208                 /* write SOA */
4209                 sldns_buffer_write_u8(buf, 0xC0); /* compressed ptr to qname */
4210                 sldns_buffer_write_u8(buf, 0x0C);
4211                 sldns_buffer_write_u16(buf, LDNS_RR_TYPE_SOA);
4212                 sldns_buffer_write_u16(buf, qinfo.qclass);
4213                 sldns_buffer_write_u32(buf, 0); /* ttl */
4214                 sldns_buffer_write_u16(buf, 22); /* rdata length */
4215                 sldns_buffer_write_u8(buf, 0); /* . */
4216                 sldns_buffer_write_u8(buf, 0); /* . */
4217                 sldns_buffer_write_u32(buf, serial); /* serial */
4218                 sldns_buffer_write_u32(buf, 0); /* refresh */
4219                 sldns_buffer_write_u32(buf, 0); /* retry */
4220                 sldns_buffer_write_u32(buf, 0); /* expire */
4221                 sldns_buffer_write_u32(buf, 0); /* minimum */
4222                 sldns_buffer_flip(buf);
4223         }
4224 }
4225
4226 /** check if returned packet is OK */
4227 static int
4228 check_packet_ok(sldns_buffer* pkt, uint16_t qtype, struct auth_xfer* xfr,
4229         uint32_t* serial)
4230 {
4231         /* parse to see if packet worked, valid reply */
4232
4233         /* check serial number of SOA */
4234         if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE)
4235                 return 0;
4236
4237         /* check ID */
4238         if(LDNS_ID_WIRE(sldns_buffer_begin(pkt)) != xfr->task_probe->id)
4239                 return 0;
4240
4241         /* check flag bits and rcode */
4242         if(!LDNS_QR_WIRE(sldns_buffer_begin(pkt)))
4243                 return 0;
4244         if(LDNS_OPCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_PACKET_QUERY)
4245                 return 0;
4246         if(LDNS_RCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_RCODE_NOERROR)
4247                 return 0;
4248
4249         /* check qname */
4250         if(LDNS_QDCOUNT(sldns_buffer_begin(pkt)) != 1)
4251                 return 0;
4252         sldns_buffer_skip(pkt, LDNS_HEADER_SIZE);
4253         if(sldns_buffer_remaining(pkt) < xfr->namelen)
4254                 return 0;
4255         if(query_dname_compare(sldns_buffer_current(pkt), xfr->name) != 0)
4256                 return 0;
4257         sldns_buffer_skip(pkt, (ssize_t)xfr->namelen);
4258
4259         /* check qtype, qclass */
4260         if(sldns_buffer_remaining(pkt) < 4)
4261                 return 0;
4262         if(sldns_buffer_read_u16(pkt) != qtype)
4263                 return 0;
4264         if(sldns_buffer_read_u16(pkt) != xfr->dclass)
4265                 return 0;
4266
4267         if(serial) {
4268                 uint16_t rdlen;
4269                 /* read serial number, from answer section SOA */
4270                 if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0)
4271                         return 0;
4272                 /* read from first record SOA record */
4273                 if(sldns_buffer_remaining(pkt) < 1)
4274                         return 0;
4275                 if(dname_pkt_compare(pkt, sldns_buffer_current(pkt),
4276                         xfr->name) != 0)
4277                         return 0;
4278                 if(!pkt_dname_len(pkt))
4279                         return 0;
4280                 /* type, class, ttl, rdatalen */
4281                 if(sldns_buffer_remaining(pkt) < 4+4+2)
4282                         return 0;
4283                 if(sldns_buffer_read_u16(pkt) != qtype)
4284                         return 0;
4285                 if(sldns_buffer_read_u16(pkt) != xfr->dclass)
4286                         return 0;
4287                 sldns_buffer_skip(pkt, 4); /* ttl */
4288                 rdlen = sldns_buffer_read_u16(pkt);
4289                 if(sldns_buffer_remaining(pkt) < rdlen)
4290                         return 0;
4291                 if(sldns_buffer_remaining(pkt) < 1)
4292                         return 0;
4293                 if(!pkt_dname_len(pkt)) /* soa name */
4294                         return 0;
4295                 if(sldns_buffer_remaining(pkt) < 1)
4296                         return 0;
4297                 if(!pkt_dname_len(pkt)) /* soa name */
4298                         return 0;
4299                 if(sldns_buffer_remaining(pkt) < 20)
4300                         return 0;
4301                 *serial = sldns_buffer_read_u32(pkt);
4302         }
4303         return 1;
4304 }
4305
4306 /** read one line from chunks into buffer at current position */
4307 static int
4308 chunkline_get_line(struct auth_chunk** chunk, size_t* chunk_pos,
4309         sldns_buffer* buf)
4310 {
4311         int readsome = 0;
4312         while(*chunk) {
4313                 /* more text in this chunk? */
4314                 if(*chunk_pos < (*chunk)->len) {
4315                         readsome = 1;
4316                         while(*chunk_pos < (*chunk)->len) {
4317                                 char c = (char)((*chunk)->data[*chunk_pos]);
4318                                 (*chunk_pos)++;
4319                                 if(sldns_buffer_remaining(buf) < 2) {
4320                                         /* buffer too short */
4321                                         verbose(VERB_ALGO, "http chunkline, "
4322                                                 "line too long");
4323                                         return 0;
4324                                 }
4325                                 sldns_buffer_write_u8(buf, (uint8_t)c);
4326                                 if(c == '\n') {
4327                                         /* we are done */
4328                                         return 1;
4329                                 }
4330                         }
4331                 }
4332                 /* move to next chunk */
4333                 *chunk = (*chunk)->next;
4334                 *chunk_pos = 0;
4335         }
4336         /* no more text */
4337         if(readsome) return 1;
4338         return 0;
4339 }
4340
4341 /** count number of open and closed parenthesis in a chunkline */
4342 static int
4343 chunkline_count_parens(sldns_buffer* buf, size_t start)
4344 {
4345         size_t end = sldns_buffer_position(buf);
4346         size_t i;
4347         int count = 0;
4348         int squote = 0, dquote = 0;
4349         for(i=start; i<end; i++) {
4350                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4351                 if(squote && c != '\'') continue;
4352                 if(dquote && c != '"') continue;
4353                 if(c == '"')
4354                         dquote = !dquote; /* skip quoted part */
4355                 else if(c == '\'')
4356                         squote = !squote; /* skip quoted part */
4357                 else if(c == '(')
4358                         count ++;
4359                 else if(c == ')')
4360                         count --;
4361                 else if(c == ';') {
4362                         /* rest is a comment */
4363                         return count;
4364                 }
4365         }
4366         return count;
4367 }
4368
4369 /** remove trailing ;... comment from a line in the chunkline buffer */
4370 static void
4371 chunkline_remove_trailcomment(sldns_buffer* buf, size_t start)
4372 {
4373         size_t end = sldns_buffer_position(buf);
4374         size_t i;
4375         int squote = 0, dquote = 0;
4376         for(i=start; i<end; i++) {
4377                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4378                 if(squote && c != '\'') continue;
4379                 if(dquote && c != '"') continue;
4380                 if(c == '"')
4381                         dquote = !dquote; /* skip quoted part */
4382                 else if(c == '\'')
4383                         squote = !squote; /* skip quoted part */
4384                 else if(c == ';') {
4385                         /* rest is a comment */
4386                         sldns_buffer_set_position(buf, i);
4387                         return;
4388                 }
4389         }
4390         /* nothing to remove */
4391 }
4392
4393 /** see if a chunkline is a comment line (or empty line) */
4394 static int
4395 chunkline_is_comment_line_or_empty(sldns_buffer* buf)
4396 {
4397         size_t i, end = sldns_buffer_limit(buf);
4398         for(i=0; i<end; i++) {
4399                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4400                 if(c == ';')
4401                         return 1; /* comment */
4402                 else if(c != ' ' && c != '\t' && c != '\r' && c != '\n')
4403                         return 0; /* not a comment */
4404         }
4405         return 1; /* empty */
4406 }
4407
4408 /** find a line with ( ) collated */
4409 static int
4410 chunkline_get_line_collated(struct auth_chunk** chunk, size_t* chunk_pos,
4411         sldns_buffer* buf)
4412 {
4413         size_t pos;
4414         int parens = 0;
4415         sldns_buffer_clear(buf);
4416         pos = sldns_buffer_position(buf);
4417         if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4418                 if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4419                         sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4420                 else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4421                 sldns_buffer_flip(buf);
4422                 return 0;
4423         }
4424         parens += chunkline_count_parens(buf, pos);
4425         while(parens > 0) {
4426                 chunkline_remove_trailcomment(buf, pos);
4427                 pos = sldns_buffer_position(buf);
4428                 if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4429                         if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4430                                 sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4431                         else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4432                         sldns_buffer_flip(buf);
4433                         return 0;
4434                 }
4435                 parens += chunkline_count_parens(buf, pos);
4436         }
4437
4438         if(sldns_buffer_remaining(buf) < 1) {
4439                 verbose(VERB_ALGO, "http chunkline: "
4440                         "line too long");
4441                 return 0;
4442         }
4443         sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4444         sldns_buffer_flip(buf);
4445         return 1;
4446 }
4447
4448 /** process $ORIGIN for http */
4449 static int
4450 http_parse_origin(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4451 {
4452         char* line = (char*)sldns_buffer_begin(buf);
4453         if(strncmp(line, "$ORIGIN", 7) == 0 &&
4454                 isspace((unsigned char)line[7])) {
4455                 int s;
4456                 pstate->origin_len = sizeof(pstate->origin);
4457                 s = sldns_str2wire_dname_buf(sldns_strip_ws(line+8),
4458                         pstate->origin, &pstate->origin_len);
4459                 if(s) pstate->origin_len = 0;
4460                 return 1;
4461         }
4462         return 0;
4463 }
4464
4465 /** process $TTL for http */
4466 static int
4467 http_parse_ttl(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4468 {
4469         char* line = (char*)sldns_buffer_begin(buf);
4470         if(strncmp(line, "$TTL", 4) == 0 &&
4471                 isspace((unsigned char)line[4])) {
4472                 const char* end = NULL;
4473                 pstate->default_ttl = sldns_str2period(
4474                         sldns_strip_ws(line+5), &end);
4475                 return 1;
4476         }
4477         return 0;
4478 }
4479
4480 /** find noncomment RR line in chunks, collates lines if ( ) format */
4481 static int
4482 chunkline_non_comment_RR(struct auth_chunk** chunk, size_t* chunk_pos,
4483         sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4484 {
4485         while(chunkline_get_line_collated(chunk, chunk_pos, buf)) {
4486                 if(chunkline_is_comment_line_or_empty(buf)) {
4487                         /* a comment, go to next line */
4488                         continue;
4489                 }
4490                 if(http_parse_origin(buf, pstate)) {
4491                         continue; /* $ORIGIN has been handled */
4492                 }
4493                 if(http_parse_ttl(buf, pstate)) {
4494                         continue; /* $TTL has been handled */
4495                 }
4496                 return 1;
4497         }
4498         /* no noncomments, fail */
4499         return 0;
4500 }
4501
4502 /** check syntax of chunklist zonefile, parse first RR, return false on
4503  * failure and return a string in the scratch buffer (first RR string)
4504  * on failure. */
4505 static int
4506 http_zonefile_syntax_check(struct auth_xfer* xfr, sldns_buffer* buf)
4507 {
4508         uint8_t rr[LDNS_RR_BUF_SIZE];
4509         size_t rr_len, dname_len = 0;
4510         struct sldns_file_parse_state pstate;
4511         struct auth_chunk* chunk;
4512         size_t chunk_pos;
4513         int e;
4514         memset(&pstate, 0, sizeof(pstate));
4515         pstate.default_ttl = 3600;
4516         if(xfr->namelen < sizeof(pstate.origin)) {
4517                 pstate.origin_len = xfr->namelen;
4518                 memmove(pstate.origin, xfr->name, xfr->namelen);
4519         }
4520         chunk = xfr->task_transfer->chunks_first;
4521         chunk_pos = 0;
4522         if(!chunkline_non_comment_RR(&chunk, &chunk_pos, buf, &pstate)) {
4523                 return 0;
4524         }
4525         rr_len = sizeof(rr);
4526         e=sldns_str2wire_rr_buf((char*)sldns_buffer_begin(buf), rr, &rr_len,
4527                 &dname_len, pstate.default_ttl,
4528                 pstate.origin_len?pstate.origin:NULL, pstate.origin_len,
4529                 pstate.prev_rr_len?pstate.prev_rr:NULL, pstate.prev_rr_len);
4530         if(e != 0) {
4531                 log_err("parse failure on first RR[%d]: %s",
4532                         LDNS_WIREPARSE_OFFSET(e),
4533                         sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)));
4534                 return 0;
4535         }
4536         /* check that class is correct */
4537         if(sldns_wirerr_get_class(rr, rr_len, dname_len) != xfr->dclass) {
4538                 log_err("parse failure: first record in downloaded zonefile "
4539                         "from wrong RR class");
4540                 return 0;
4541         }
4542         return 1;
4543 }
4544
4545 /** sum sizes of chunklist */
4546 static size_t
4547 chunklist_sum(struct auth_chunk* list)
4548 {
4549         struct auth_chunk* p;
4550         size_t s = 0;
4551         for(p=list; p; p=p->next) {
4552                 s += p->len;
4553         }
4554         return s;
4555 }
4556
4557 /** remove newlines from collated line */
4558 static void
4559 chunkline_newline_removal(sldns_buffer* buf)
4560 {
4561         size_t i, end=sldns_buffer_limit(buf);
4562         for(i=0; i<end; i++) {
4563                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4564                 if(c == '\n' && i==end-1) {
4565                         sldns_buffer_write_u8_at(buf, i, 0);
4566                         sldns_buffer_set_limit(buf, end-1);
4567                         return;
4568                 }
4569                 if(c == '\n')
4570                         sldns_buffer_write_u8_at(buf, i, (uint8_t)' ');
4571         }
4572 }
4573
4574 /** for http download, parse and add RR to zone */
4575 static int
4576 http_parse_add_rr(struct auth_xfer* xfr, struct auth_zone* z,
4577         sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4578 {
4579         uint8_t rr[LDNS_RR_BUF_SIZE];
4580         size_t rr_len, dname_len = 0;
4581         int e;
4582         char* line = (char*)sldns_buffer_begin(buf);
4583         rr_len = sizeof(rr);
4584         e = sldns_str2wire_rr_buf(line, rr, &rr_len, &dname_len,
4585                 pstate->default_ttl,
4586                 pstate->origin_len?pstate->origin:NULL, pstate->origin_len,
4587                 pstate->prev_rr_len?pstate->prev_rr:NULL, pstate->prev_rr_len);
4588         if(e != 0) {
4589                 log_err("%s/%s parse failure RR[%d]: %s in '%s'",
4590                         xfr->task_transfer->master->host,
4591                         xfr->task_transfer->master->file,
4592                         LDNS_WIREPARSE_OFFSET(e),
4593                         sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)),
4594                         line);
4595                 return 0;
4596         }
4597         if(rr_len == 0)
4598                 return 1; /* empty line or so */
4599
4600         /* set prev */
4601         if(dname_len < sizeof(pstate->prev_rr)) {
4602                 memmove(pstate->prev_rr, rr, dname_len);
4603                 pstate->prev_rr_len = dname_len;
4604         }
4605
4606         return az_insert_rr(z, rr, rr_len, dname_len, NULL);
4607 }
4608
4609 /** RR list iterator, returns RRs from answer section one by one from the
4610  * dns packets in the chunklist */
4611 static void
4612 chunk_rrlist_start(struct auth_xfer* xfr, struct auth_chunk** rr_chunk,
4613         int* rr_num, size_t* rr_pos)
4614 {
4615         *rr_chunk = xfr->task_transfer->chunks_first;
4616         *rr_num = 0;
4617         *rr_pos = 0;
4618 }
4619
4620 /** RR list iterator, see if we are at the end of the list */
4621 static int
4622 chunk_rrlist_end(struct auth_chunk* rr_chunk, int rr_num)
4623 {
4624         while(rr_chunk) {
4625                 if(rr_chunk->len < LDNS_HEADER_SIZE)
4626                         return 1;
4627                 if(rr_num < (int)LDNS_ANCOUNT(rr_chunk->data))
4628                         return 0;
4629                 /* no more RRs in this chunk */
4630                 /* continue with next chunk, see if it has RRs */
4631                 rr_chunk = rr_chunk->next;
4632                 rr_num = 0;
4633         }
4634         return 1;
4635 }
4636
4637 /** RR list iterator, move to next RR */
4638 static void
4639 chunk_rrlist_gonext(struct auth_chunk** rr_chunk, int* rr_num,
4640         size_t* rr_pos, size_t rr_nextpos)
4641 {
4642         /* already at end of chunks? */
4643         if(!*rr_chunk)
4644                 return;
4645         /* move within this chunk */
4646         if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4647                 (*rr_num)+1 < (int)LDNS_ANCOUNT((*rr_chunk)->data)) {
4648                 (*rr_num) += 1;
4649                 *rr_pos = rr_nextpos;
4650                 return;
4651         }
4652         /* no more RRs in this chunk */
4653         /* continue with next chunk, see if it has RRs */
4654         if(*rr_chunk)
4655                 *rr_chunk = (*rr_chunk)->next;
4656         while(*rr_chunk) {
4657                 *rr_num = 0;
4658                 *rr_pos = 0;
4659                 if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4660                         LDNS_ANCOUNT((*rr_chunk)->data) > 0) {
4661                         return;
4662                 }
4663                 *rr_chunk = (*rr_chunk)->next;
4664         }
4665 }
4666
4667 /** RR iterator, get current RR information, false on parse error */
4668 static int
4669 chunk_rrlist_get_current(struct auth_chunk* rr_chunk, int rr_num,
4670         size_t rr_pos, uint8_t** rr_dname, uint16_t* rr_type,
4671         uint16_t* rr_class, uint32_t* rr_ttl, uint16_t* rr_rdlen,
4672         uint8_t** rr_rdata, size_t* rr_nextpos)
4673 {
4674         sldns_buffer pkt;
4675         /* integrity checks on position */
4676         if(!rr_chunk) return 0;
4677         if(rr_chunk->len < LDNS_HEADER_SIZE) return 0;
4678         if(rr_num >= (int)LDNS_ANCOUNT(rr_chunk->data)) return 0;
4679         if(rr_pos >= rr_chunk->len) return 0;
4680
4681         /* fetch rr information */
4682         sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4683         if(rr_pos == 0) {
4684                 size_t i;
4685                 /* skip question section */
4686                 sldns_buffer_set_position(&pkt, LDNS_HEADER_SIZE);
4687                 for(i=0; i<LDNS_QDCOUNT(rr_chunk->data); i++) {
4688                         if(pkt_dname_len(&pkt) == 0) return 0;
4689                         if(sldns_buffer_remaining(&pkt) < 4) return 0;
4690                         sldns_buffer_skip(&pkt, 4); /* type and class */
4691                 }
4692         } else  {
4693                 sldns_buffer_set_position(&pkt, rr_pos);
4694         }
4695         *rr_dname = sldns_buffer_current(&pkt);
4696         if(pkt_dname_len(&pkt) == 0) return 0;
4697         if(sldns_buffer_remaining(&pkt) < 10) return 0;
4698         *rr_type = sldns_buffer_read_u16(&pkt);
4699         *rr_class = sldns_buffer_read_u16(&pkt);
4700         *rr_ttl = sldns_buffer_read_u32(&pkt);
4701         *rr_rdlen = sldns_buffer_read_u16(&pkt);
4702         if(sldns_buffer_remaining(&pkt) < (*rr_rdlen)) return 0;
4703         *rr_rdata = sldns_buffer_current(&pkt);
4704         sldns_buffer_skip(&pkt, (ssize_t)(*rr_rdlen));
4705         *rr_nextpos = sldns_buffer_position(&pkt);
4706         return 1;
4707 }
4708
4709 /** print log message where we are in parsing the zone transfer */
4710 static void
4711 log_rrlist_position(const char* label, struct auth_chunk* rr_chunk,
4712         uint8_t* rr_dname, uint16_t rr_type, size_t rr_counter)
4713 {
4714         sldns_buffer pkt;
4715         size_t dlen;
4716         uint8_t buf[256];
4717         char str[256];
4718         char typestr[32];
4719         sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4720         sldns_buffer_set_position(&pkt, (size_t)(rr_dname -
4721                 sldns_buffer_begin(&pkt)));
4722         if((dlen=pkt_dname_len(&pkt)) == 0) return;
4723         if(dlen >= sizeof(buf)) return;
4724         dname_pkt_copy(&pkt, buf, rr_dname);
4725         dname_str(buf, str);
4726         (void)sldns_wire2str_type_buf(rr_type, typestr, sizeof(typestr));
4727         verbose(VERB_ALGO, "%s at[%d] %s %s", label, (int)rr_counter,
4728                 str, typestr);
4729 }
4730
4731 /** check that start serial is OK for ixfr. we are at rr_counter == 0,
4732  * and we are going to check rr_counter == 1 (has to be type SOA) serial */
4733 static int
4734 ixfr_start_serial(struct auth_chunk* rr_chunk, int rr_num, size_t rr_pos,
4735         uint8_t* rr_dname, uint16_t rr_type, uint16_t rr_class,
4736         uint32_t rr_ttl, uint16_t rr_rdlen, uint8_t* rr_rdata,
4737         size_t rr_nextpos, uint32_t transfer_serial, uint32_t xfr_serial)
4738 {
4739         uint32_t startserial;
4740         /* move forward on RR */
4741         chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4742         if(chunk_rrlist_end(rr_chunk, rr_num)) {
4743                 /* no second SOA */
4744                 verbose(VERB_OPS, "IXFR has no second SOA record");
4745                 return 0;
4746         }
4747         if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4748                 &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4749                 &rr_rdata, &rr_nextpos)) {
4750                 verbose(VERB_OPS, "IXFR cannot parse second SOA record");
4751                 /* failed to parse RR */
4752                 return 0;
4753         }
4754         if(rr_type != LDNS_RR_TYPE_SOA) {
4755                 verbose(VERB_OPS, "IXFR second record is not type SOA");
4756                 return 0;
4757         }
4758         if(rr_rdlen < 22) {
4759                 verbose(VERB_OPS, "IXFR, second SOA has short rdlength");
4760                 return 0; /* bad SOA rdlen */
4761         }
4762         startserial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4763         if(startserial == transfer_serial) {
4764                 /* empty AXFR, not an IXFR */
4765                 verbose(VERB_OPS, "IXFR second serial same as first");
4766                 return 0;
4767         }
4768         if(startserial != xfr_serial) {
4769                 /* wrong start serial, it does not match the serial in
4770                  * memory */
4771                 verbose(VERB_OPS, "IXFR is from serial %u to %u but %u "
4772                         "in memory, rejecting the zone transfer",
4773                         (unsigned)startserial, (unsigned)transfer_serial,
4774                         (unsigned)xfr_serial);
4775                 return 0;
4776         }
4777         /* everything OK in second SOA serial */
4778         return 1;
4779 }
4780
4781 /** apply IXFR to zone in memory. z is locked. false on failure(mallocfail) */
4782 static int
4783 apply_ixfr(struct auth_xfer* xfr, struct auth_zone* z,
4784         struct sldns_buffer* scratch_buffer)
4785 {
4786         struct auth_chunk* rr_chunk;
4787         int rr_num;
4788         size_t rr_pos;
4789         uint8_t* rr_dname, *rr_rdata;
4790         uint16_t rr_type, rr_class, rr_rdlen;
4791         uint32_t rr_ttl;
4792         size_t rr_nextpos;
4793         int have_transfer_serial = 0;
4794         uint32_t transfer_serial = 0;
4795         size_t rr_counter = 0;
4796         int delmode = 0;
4797         int softfail = 0;
4798
4799         /* start RR iterator over chunklist of packets */
4800         chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4801         while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4802                 if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4803                         &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4804                         &rr_rdata, &rr_nextpos)) {
4805                         /* failed to parse RR */
4806                         return 0;
4807                 }
4808                 if(verbosity>=7) log_rrlist_position("apply ixfr",
4809                         rr_chunk, rr_dname, rr_type, rr_counter);
4810                 /* twiddle add/del mode and check for start and end */
4811                 if(rr_counter == 0 && rr_type != LDNS_RR_TYPE_SOA)
4812                         return 0;
4813                 if(rr_counter == 1 && rr_type != LDNS_RR_TYPE_SOA) {
4814                         /* this is an AXFR returned from the IXFR master */
4815                         /* but that should already have been detected, by
4816                          * on_ixfr_is_axfr */
4817                         return 0;
4818                 }
4819                 if(rr_type == LDNS_RR_TYPE_SOA) {
4820                         uint32_t serial;
4821                         if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4822                         serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4823                         if(have_transfer_serial == 0) {
4824                                 have_transfer_serial = 1;
4825                                 transfer_serial = serial;
4826                                 delmode = 1; /* gets negated below */
4827                                 /* check second RR before going any further */
4828                                 if(!ixfr_start_serial(rr_chunk, rr_num, rr_pos,
4829                                         rr_dname, rr_type, rr_class, rr_ttl,
4830                                         rr_rdlen, rr_rdata, rr_nextpos,
4831                                         transfer_serial, xfr->serial)) {
4832                                         return 0;
4833                                 }
4834                         } else if(transfer_serial == serial) {
4835                                 have_transfer_serial++;
4836                                 if(rr_counter == 1) {
4837                                         /* empty AXFR, with SOA; SOA; */
4838                                         /* should have been detected by
4839                                          * on_ixfr_is_axfr */
4840                                         return 0;
4841                                 }
4842                                 if(have_transfer_serial == 3) {
4843                                         /* see serial three times for end */
4844                                         /* eg. IXFR:
4845                                          *  SOA 3 start
4846                                          *  SOA 1 second RR, followed by del
4847                                          *  SOA 2 followed by add
4848                                          *  SOA 2 followed by del
4849                                          *  SOA 3 followed by add
4850                                          *  SOA 3 end */
4851                                         /* ended by SOA record */
4852                                         xfr->serial = transfer_serial;
4853                                         break;
4854                                 }
4855                         }
4856                         /* twiddle add/del mode */
4857                         /* switch from delete part to add part and back again
4858                          * just before the soa, it gets deleted and added too
4859                          * this means we switch to delete mode for the final
4860                          * SOA(so skip that one) */
4861                         delmode = !delmode;
4862                 }
4863                 /* process this RR */
4864                 /* if the RR is deleted twice or added twice, then we
4865                  * softfail, and continue with the rest of the IXFR, so
4866                  * that we serve something fairly nice during the refetch */
4867                 if(verbosity>=7) log_rrlist_position((delmode?"del":"add"),
4868                         rr_chunk, rr_dname, rr_type, rr_counter);
4869                 if(delmode) {
4870                         /* delete this RR */
4871                         int nonexist = 0;
4872                         if(!az_remove_rr_decompress(z, rr_chunk->data,
4873                                 rr_chunk->len, scratch_buffer, rr_dname,
4874                                 rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4875                                 &nonexist)) {
4876                                 /* failed, malloc error or so */
4877                                 return 0;
4878                         }
4879                         if(nonexist) {
4880                                 /* it was removal of a nonexisting RR */
4881                                 if(verbosity>=4) log_rrlist_position(
4882                                         "IXFR error nonexistent RR",
4883                                         rr_chunk, rr_dname, rr_type, rr_counter);
4884                                 softfail = 1;
4885                         }
4886                 } else if(rr_counter != 0) {
4887                         /* skip first SOA RR for addition, it is added in
4888                          * the addition part near the end of the ixfr, when
4889                          * that serial is seen the second time. */
4890                         int duplicate = 0;
4891                         /* add this RR */
4892                         if(!az_insert_rr_decompress(z, rr_chunk->data,
4893                                 rr_chunk->len, scratch_buffer, rr_dname,
4894                                 rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4895                                 &duplicate)) {
4896                                 /* failed, malloc error or so */
4897                                 return 0;
4898                         }
4899                         if(duplicate) {
4900                                 /* it was a duplicate */
4901                                 if(verbosity>=4) log_rrlist_position(
4902                                         "IXFR error duplicate RR",
4903                                         rr_chunk, rr_dname, rr_type, rr_counter);
4904                                 softfail = 1;
4905                         }
4906                 }
4907
4908                 rr_counter++;
4909                 chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4910         }
4911         if(softfail) {
4912                 verbose(VERB_ALGO, "IXFR did not apply cleanly, fetching full zone");
4913                 return 0;
4914         }
4915         return 1;
4916 }
4917
4918 /** apply AXFR to zone in memory. z is locked. false on failure(mallocfail) */
4919 static int
4920 apply_axfr(struct auth_xfer* xfr, struct auth_zone* z,
4921         struct sldns_buffer* scratch_buffer)
4922 {
4923         struct auth_chunk* rr_chunk;
4924         int rr_num;
4925         size_t rr_pos;
4926         uint8_t* rr_dname, *rr_rdata;
4927         uint16_t rr_type, rr_class, rr_rdlen;
4928         uint32_t rr_ttl;
4929         uint32_t serial = 0;
4930         size_t rr_nextpos;
4931         size_t rr_counter = 0;
4932         int have_end_soa = 0;
4933
4934         /* clear the data tree */
4935         traverse_postorder(&z->data, auth_data_del, NULL);
4936         rbtree_init(&z->data, &auth_data_cmp);
4937         /* clear the RPZ policies */
4938         if(z->rpz)
4939                 rpz_clear(z->rpz);
4940
4941         xfr->have_zone = 0;
4942         xfr->serial = 0;
4943
4944         /* insert all RRs in to the zone */
4945         /* insert the SOA only once, skip the last one */
4946         /* start RR iterator over chunklist of packets */
4947         chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4948         while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4949                 if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4950                         &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4951                         &rr_rdata, &rr_nextpos)) {
4952                         /* failed to parse RR */
4953                         return 0;
4954                 }
4955                 if(verbosity>=7) log_rrlist_position("apply_axfr",
4956                         rr_chunk, rr_dname, rr_type, rr_counter);
4957                 if(rr_type == LDNS_RR_TYPE_SOA) {
4958                         if(rr_counter != 0) {
4959                                 /* end of the axfr */
4960                                 have_end_soa = 1;
4961                                 break;
4962                         }
4963                         if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4964                         serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4965                 }
4966
4967                 /* add this RR */
4968                 if(!az_insert_rr_decompress(z, rr_chunk->data, rr_chunk->len,
4969                         scratch_buffer, rr_dname, rr_type, rr_class, rr_ttl,
4970                         rr_rdata, rr_rdlen, NULL)) {
4971                         /* failed, malloc error or so */
4972                         return 0;
4973                 }
4974
4975                 rr_counter++;
4976                 chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4977         }
4978         if(!have_end_soa) {
4979                 log_err("no end SOA record for AXFR");
4980                 return 0;
4981         }
4982
4983         xfr->serial = serial;
4984         xfr->have_zone = 1;
4985         return 1;
4986 }
4987
4988 /** apply HTTP to zone in memory. z is locked. false on failure(mallocfail) */
4989 static int
4990 apply_http(struct auth_xfer* xfr, struct auth_zone* z,
4991         struct sldns_buffer* scratch_buffer)
4992 {
4993         /* parse data in chunks */
4994         /* parse RR's and read into memory. ignore $INCLUDE from the
4995          * downloaded file*/
4996         struct sldns_file_parse_state pstate;
4997         struct auth_chunk* chunk;
4998         size_t chunk_pos;
4999         memset(&pstate, 0, sizeof(pstate));
5000         pstate.default_ttl = 3600;
5001         if(xfr->namelen < sizeof(pstate.origin)) {
5002                 pstate.origin_len = xfr->namelen;
5003                 memmove(pstate.origin, xfr->name, xfr->namelen);
5004         }
5005
5006         if(verbosity >= VERB_ALGO)
5007                 verbose(VERB_ALGO, "http download %s of size %d",
5008                 xfr->task_transfer->master->file,
5009                 (int)chunklist_sum(xfr->task_transfer->chunks_first));
5010         if(xfr->task_transfer->chunks_first && verbosity >= VERB_ALGO) {
5011                 char preview[1024];
5012                 if(xfr->task_transfer->chunks_first->len+1 > sizeof(preview)) {
5013                         memmove(preview, xfr->task_transfer->chunks_first->data,
5014                                 sizeof(preview)-1);
5015                         preview[sizeof(preview)-1]=0;
5016                 } else {
5017                         memmove(preview, xfr->task_transfer->chunks_first->data,
5018                                 xfr->task_transfer->chunks_first->len);
5019                         preview[xfr->task_transfer->chunks_first->len]=0;
5020                 }
5021                 log_info("auth zone http downloaded content preview: %s",
5022                         preview);
5023         }
5024
5025         /* perhaps a little syntax check before we try to apply the data? */
5026         if(!http_zonefile_syntax_check(xfr, scratch_buffer)) {
5027                 log_err("http download %s/%s does not contain a zonefile, "
5028                         "but got '%s'", xfr->task_transfer->master->host,
5029                         xfr->task_transfer->master->file,
5030                         sldns_buffer_begin(scratch_buffer));
5031                 return 0;
5032         }
5033
5034         /* clear the data tree */
5035         traverse_postorder(&z->data, auth_data_del, NULL);
5036         rbtree_init(&z->data, &auth_data_cmp);
5037         /* clear the RPZ policies */
5038         if(z->rpz)
5039                 rpz_clear(z->rpz);
5040
5041         xfr->have_zone = 0;
5042         xfr->serial = 0;
5043
5044         chunk = xfr->task_transfer->chunks_first;
5045         chunk_pos = 0;
5046         pstate.lineno = 0;
5047         while(chunkline_get_line_collated(&chunk, &chunk_pos, scratch_buffer)) {
5048                 /* process this line */
5049                 pstate.lineno++;
5050                 chunkline_newline_removal(scratch_buffer);
5051                 if(chunkline_is_comment_line_or_empty(scratch_buffer)) {
5052                         continue;
5053                 }
5054                 /* parse line and add RR */
5055                 if(http_parse_origin(scratch_buffer, &pstate)) {
5056                         continue; /* $ORIGIN has been handled */
5057                 }
5058                 if(http_parse_ttl(scratch_buffer, &pstate)) {
5059                         continue; /* $TTL has been handled */
5060                 }
5061                 if(!http_parse_add_rr(xfr, z, scratch_buffer, &pstate)) {
5062                         verbose(VERB_ALGO, "error parsing line [%s:%d] %s",
5063                                 xfr->task_transfer->master->file,
5064                                 pstate.lineno,
5065                                 sldns_buffer_begin(scratch_buffer));
5066                         return 0;
5067                 }
5068         }
5069         return 1;
5070 }
5071
5072 /** write http chunks to zonefile to create downloaded file */
5073 static int
5074 auth_zone_write_chunks(struct auth_xfer* xfr, const char* fname)
5075 {
5076         FILE* out;
5077         struct auth_chunk* p;
5078         out = fopen(fname, "w");
5079         if(!out) {
5080                 log_err("could not open %s: %s", fname, strerror(errno));
5081                 return 0;
5082         }
5083         for(p = xfr->task_transfer->chunks_first; p ; p = p->next) {
5084                 if(!write_out(out, (char*)p->data, p->len)) {
5085                         log_err("could not write http download to %s", fname);
5086                         fclose(out);
5087                         return 0;
5088                 }
5089         }
5090         fclose(out);
5091         return 1;
5092 }
5093
5094 /** write to zonefile after zone has been updated */
5095 static void
5096 xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
5097 {
5098         struct config_file* cfg = env->cfg;
5099         struct auth_zone* z;
5100         char tmpfile[1024];
5101         char* zfilename;
5102         lock_basic_unlock(&xfr->lock);
5103
5104         /* get lock again, so it is a readlock and concurrently queries
5105          * can be answered */
5106         lock_rw_rdlock(&env->auth_zones->lock);
5107         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
5108                 xfr->dclass);
5109         if(!z) {
5110                 lock_rw_unlock(&env->auth_zones->lock);
5111                 /* the zone is gone, ignore xfr results */
5112                 lock_basic_lock(&xfr->lock);
5113                 return;
5114         }
5115         lock_rw_rdlock(&z->lock);
5116         lock_basic_lock(&xfr->lock);
5117         lock_rw_unlock(&env->auth_zones->lock);
5118
5119         if(z->zonefile == NULL || z->zonefile[0] == 0) {
5120                 lock_rw_unlock(&z->lock);
5121                 /* no write needed, no zonefile set */
5122                 return;
5123         }
5124         zfilename = z->zonefile;
5125         if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
5126                 cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
5127                 zfilename += strlen(cfg->chrootdir);
5128         if(verbosity >= VERB_ALGO) {
5129                 char nm[255+1];
5130                 dname_str(z->name, nm);
5131                 verbose(VERB_ALGO, "write zonefile %s for %s", zfilename, nm);
5132         }
5133
5134         /* write to tempfile first */
5135         if((size_t)strlen(zfilename) + 16 > sizeof(tmpfile)) {
5136                 verbose(VERB_ALGO, "tmpfilename too long, cannot update "
5137                         " zonefile %s", zfilename);
5138                 lock_rw_unlock(&z->lock);
5139                 return;
5140         }
5141         snprintf(tmpfile, sizeof(tmpfile), "%s.tmp%u", zfilename,
5142                 (unsigned)getpid());
5143         if(xfr->task_transfer->master->http) {
5144                 /* use the stored chunk list to write them */
5145                 if(!auth_zone_write_chunks(xfr, tmpfile)) {
5146                         unlink(tmpfile);
5147                         lock_rw_unlock(&z->lock);
5148                         return;
5149                 }
5150         } else if(!auth_zone_write_file(z, tmpfile)) {
5151                 unlink(tmpfile);
5152                 lock_rw_unlock(&z->lock);
5153                 return;
5154         }
5155 #ifdef UB_ON_WINDOWS
5156         (void)unlink(zfilename); /* windows does not replace file with rename() */
5157 #endif
5158         if(rename(tmpfile, zfilename) < 0) {
5159                 log_err("could not rename(%s, %s): %s", tmpfile, zfilename,
5160                         strerror(errno));
5161                 unlink(tmpfile);
5162                 lock_rw_unlock(&z->lock);
5163                 return;
5164         }
5165         lock_rw_unlock(&z->lock);
5166 }
5167
5168 /** reacquire locks and structures. Starts with no locks, ends
5169  * with xfr and z locks, if fail, no z lock */
5170 static int xfr_process_reacquire_locks(struct auth_xfer* xfr,
5171         struct module_env* env, struct auth_zone** z)
5172 {
5173         /* release xfr lock, then, while holding az->lock grab both
5174          * z->lock and xfr->lock */
5175         lock_rw_rdlock(&env->auth_zones->lock);
5176         *z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
5177                 xfr->dclass);
5178         if(!*z) {
5179                 lock_rw_unlock(&env->auth_zones->lock);
5180                 lock_basic_lock(&xfr->lock);
5181                 *z = NULL;
5182                 return 0;
5183         }
5184         lock_rw_wrlock(&(*z)->lock);
5185         lock_basic_lock(&xfr->lock);
5186         lock_rw_unlock(&env->auth_zones->lock);
5187         return 1;
5188 }
5189
5190 /** process chunk list and update zone in memory,
5191  * return false if it did not work */
5192 static int
5193 xfr_process_chunk_list(struct auth_xfer* xfr, struct module_env* env,
5194         int* ixfr_fail)
5195 {
5196         struct auth_zone* z;
5197
5198         /* obtain locks and structures */
5199         lock_basic_unlock(&xfr->lock);
5200         if(!xfr_process_reacquire_locks(xfr, env, &z)) {
5201                 /* the zone is gone, ignore xfr results */
5202                 return 0;
5203         }
5204         /* holding xfr and z locks */
5205
5206         /* apply data */
5207         if(xfr->task_transfer->master->http) {
5208                 if(!apply_http(xfr, z, env->scratch_buffer)) {
5209                         lock_rw_unlock(&z->lock);
5210                         verbose(VERB_ALGO, "http from %s: could not store data",
5211                                 xfr->task_transfer->master->host);
5212                         return 0;
5213                 }
5214         } else if(xfr->task_transfer->on_ixfr &&
5215                 !xfr->task_transfer->on_ixfr_is_axfr) {
5216                 if(!apply_ixfr(xfr, z, env->scratch_buffer)) {
5217                         lock_rw_unlock(&z->lock);
5218                         verbose(VERB_ALGO, "xfr from %s: could not store IXFR"
5219                                 " data", xfr->task_transfer->master->host);
5220                         *ixfr_fail = 1;
5221                         return 0;
5222                 }
5223         } else {
5224                 if(!apply_axfr(xfr, z, env->scratch_buffer)) {
5225                         lock_rw_unlock(&z->lock);
5226                         verbose(VERB_ALGO, "xfr from %s: could not store AXFR"
5227                                 " data", xfr->task_transfer->master->host);
5228                         return 0;
5229                 }
5230         }
5231         xfr->zone_expired = 0;
5232         z->zone_expired = 0;
5233         if(!xfr_find_soa(z, xfr)) {
5234                 lock_rw_unlock(&z->lock);
5235                 verbose(VERB_ALGO, "xfr from %s: no SOA in zone after update"
5236                         " (or malformed RR)", xfr->task_transfer->master->host);
5237                 return 0;
5238         }
5239
5240         /* release xfr lock while verifying zonemd because it may have
5241          * to spawn lookups in the state machines */
5242         lock_basic_unlock(&xfr->lock);
5243         /* holding z lock */
5244         auth_zone_verify_zonemd(z, env, &env->mesh->mods, NULL, 0, 0);
5245         if(z->zone_expired) {
5246                 char zname[256];
5247                 /* ZONEMD must have failed */
5248                 /* reacquire locks, so we hold xfr lock on exit of routine,
5249                  * and both xfr and z again after releasing xfr for potential
5250                  * state machine mesh callbacks */
5251                 lock_rw_unlock(&z->lock);
5252                 if(!xfr_process_reacquire_locks(xfr, env, &z))
5253                         return 0;
5254                 dname_str(xfr->name, zname);
5255                 verbose(VERB_ALGO, "xfr from %s: ZONEMD failed for %s, transfer is failed", xfr->task_transfer->master->host, zname);
5256                 xfr->zone_expired = 1;
5257                 lock_rw_unlock(&z->lock);
5258                 return 0;
5259         }
5260         /* reacquire locks, so we hold xfr lock on exit of routine,
5261          * and both xfr and z again after releasing xfr for potential
5262          * state machine mesh callbacks */
5263         lock_rw_unlock(&z->lock);
5264         if(!xfr_process_reacquire_locks(xfr, env, &z))
5265                 return 0;
5266         /* holding xfr and z locks */
5267
5268         if(xfr->have_zone)
5269                 xfr->lease_time = *env->now;
5270
5271         if(z->rpz)
5272                 rpz_finish_config(z->rpz);
5273
5274         /* unlock */
5275         lock_rw_unlock(&z->lock);
5276
5277         if(verbosity >= VERB_QUERY && xfr->have_zone) {
5278                 char zname[256];
5279                 dname_str(xfr->name, zname);
5280                 verbose(VERB_QUERY, "auth zone %s updated to serial %u", zname,
5281                         (unsigned)xfr->serial);
5282         }
5283         /* see if we need to write to a zonefile */
5284         xfr_write_after_update(xfr, env);
5285         return 1;
5286 }
5287
5288 /** disown task_transfer.  caller must hold xfr.lock */
5289 static void
5290 xfr_transfer_disown(struct auth_xfer* xfr)
5291 {
5292         /* remove timer (from this worker's event base) */
5293         comm_timer_delete(xfr->task_transfer->timer);
5294         xfr->task_transfer->timer = NULL;
5295         /* remove the commpoint */
5296         comm_point_delete(xfr->task_transfer->cp);
5297         xfr->task_transfer->cp = NULL;
5298         /* we don't own this item anymore */
5299         xfr->task_transfer->worker = NULL;
5300         xfr->task_transfer->env = NULL;
5301 }
5302
5303 /** lookup a host name for its addresses, if needed */
5304 static int
5305 xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
5306 {
5307         struct sockaddr_storage addr;
5308         socklen_t addrlen = 0;
5309         struct auth_master* master = xfr->task_transfer->lookup_target;
5310         struct query_info qinfo;
5311         uint16_t qflags = BIT_RD;
5312         uint8_t dname[LDNS_MAX_DOMAINLEN+1];
5313         struct edns_data edns;
5314         sldns_buffer* buf = env->scratch_buffer;
5315         if(!master) return 0;
5316         if(extstrtoaddr(master->host, &addr, &addrlen)) {
5317                 /* not needed, host is in IP addr format */
5318                 return 0;
5319         }
5320         if(master->allow_notify)
5321                 return 0; /* allow-notifies are not transferred from, no
5322                 lookup is needed */
5323
5324         /* use mesh_new_callback to probe for non-addr hosts,
5325          * and then wait for them to be looked up (in cache, or query) */
5326         qinfo.qname_len = sizeof(dname);
5327         if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
5328                 != 0) {
5329                 log_err("cannot parse host name of master %s", master->host);
5330                 return 0;
5331         }
5332         qinfo.qname = dname;
5333         qinfo.qclass = xfr->dclass;
5334         qinfo.qtype = LDNS_RR_TYPE_A;
5335         if(xfr->task_transfer->lookup_aaaa)
5336                 qinfo.qtype = LDNS_RR_TYPE_AAAA;
5337         qinfo.local_alias = NULL;
5338         if(verbosity >= VERB_ALGO) {
5339                 char buf1[512];
5340                 char buf2[LDNS_MAX_DOMAINLEN+1];
5341                 dname_str(xfr->name, buf2);
5342                 snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
5343                         " for task_transfer", buf2);
5344                 log_query_info(VERB_ALGO, buf1, &qinfo);
5345         }
5346         edns.edns_present = 1;
5347         edns.ext_rcode = 0;
5348         edns.edns_version = 0;
5349         edns.bits = EDNS_DO;
5350         edns.opt_list = NULL;
5351         edns.padding_block_size = 0;
5352         if(sldns_buffer_capacity(buf) < 65535)
5353                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
5354         else    edns.udp_size = 65535;
5355
5356         /* unlock xfr during mesh_new_callback() because the callback can be
5357          * called straight away */
5358         lock_basic_unlock(&xfr->lock);
5359         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
5360                 &auth_xfer_transfer_lookup_callback, xfr)) {
5361                 lock_basic_lock(&xfr->lock);
5362                 log_err("out of memory lookup up master %s", master->host);
5363                 return 0;
5364         }
5365         lock_basic_lock(&xfr->lock);
5366         return 1;
5367 }
5368
5369 /** initiate TCP to the target and fetch zone.
5370  * returns true if that was successfully started, and timeout setup. */
5371 static int
5372 xfr_transfer_init_fetch(struct auth_xfer* xfr, struct module_env* env)
5373 {
5374         struct sockaddr_storage addr;
5375         socklen_t addrlen = 0;
5376         struct auth_master* master = xfr->task_transfer->master;
5377         char *auth_name = NULL;
5378         struct timeval t;
5379         int timeout;
5380         if(!master) return 0;
5381         if(master->allow_notify) return 0; /* only for notify */
5382
5383         /* get master addr */
5384         if(xfr->task_transfer->scan_addr) {
5385                 addrlen = xfr->task_transfer->scan_addr->addrlen;
5386                 memmove(&addr, &xfr->task_transfer->scan_addr->addr, addrlen);
5387         } else {
5388                 if(!authextstrtoaddr(master->host, &addr, &addrlen, &auth_name)) {
5389                         /* the ones that are not in addr format are supposed
5390                          * to be looked up.  The lookup has failed however,
5391                          * so skip them */
5392                         char zname[255+1];
5393                         dname_str(xfr->name, zname);
5394                         log_err("%s: failed lookup, cannot transfer from master %s",
5395                                 zname, master->host);
5396                         return 0;
5397                 }
5398         }
5399
5400         /* remove previous TCP connection (if any) */
5401         if(xfr->task_transfer->cp) {
5402                 comm_point_delete(xfr->task_transfer->cp);
5403                 xfr->task_transfer->cp = NULL;
5404         }
5405         if(!xfr->task_transfer->timer) {
5406                 xfr->task_transfer->timer = comm_timer_create(env->worker_base,
5407                         auth_xfer_transfer_timer_callback, xfr);
5408                 if(!xfr->task_transfer->timer) {
5409                         log_err("malloc failure");
5410                         return 0;
5411                 }
5412         }
5413         timeout = AUTH_TRANSFER_TIMEOUT;
5414 #ifndef S_SPLINT_S
5415         t.tv_sec = timeout/1000;
5416         t.tv_usec = (timeout%1000)*1000;
5417 #endif
5418
5419         if(master->http) {
5420                 /* perform http fetch */
5421                 /* store http port number into sockaddr,
5422                  * unless someone used unbound's host@port notation */
5423                 xfr->task_transfer->on_ixfr = 0;
5424                 if(strchr(master->host, '@') == NULL)
5425                         sockaddr_store_port(&addr, addrlen, master->port);
5426                 xfr->task_transfer->cp = outnet_comm_point_for_http(
5427                         env->outnet, auth_xfer_transfer_http_callback, xfr,
5428                         &addr, addrlen, -1, master->ssl, master->host,
5429                         master->file, env->cfg);
5430                 if(!xfr->task_transfer->cp) {
5431                         char zname[255+1], as[256];
5432                         dname_str(xfr->name, zname);
5433                         addr_to_str(&addr, addrlen, as, sizeof(as));
5434                         verbose(VERB_ALGO, "cannot create http cp "
5435                                 "connection for %s to %s", zname, as);
5436                         return 0;
5437                 }
5438                 comm_timer_set(xfr->task_transfer->timer, &t);
5439                 if(verbosity >= VERB_ALGO) {
5440                         char zname[255+1], as[256];
5441                         dname_str(xfr->name, zname);
5442                         addr_to_str(&addr, addrlen, as, sizeof(as));
5443                         verbose(VERB_ALGO, "auth zone %s transfer next HTTP fetch from %s started", zname, as);
5444                 }
5445                 return 1;
5446         }
5447
5448         /* perform AXFR/IXFR */
5449         /* set the packet to be written */
5450         /* create new ID */
5451         xfr->task_transfer->id = GET_RANDOM_ID(env->rnd);
5452         xfr_create_ixfr_packet(xfr, env->scratch_buffer,
5453                 xfr->task_transfer->id, master);
5454
5455         /* connect on fd */
5456         xfr->task_transfer->cp = outnet_comm_point_for_tcp(env->outnet,
5457                 auth_xfer_transfer_tcp_callback, xfr, &addr, addrlen,
5458                 env->scratch_buffer, -1,
5459                 auth_name != NULL, auth_name);
5460         if(!xfr->task_transfer->cp) {
5461                 char zname[255+1], as[256];
5462                 dname_str(xfr->name, zname);
5463                 addr_to_str(&addr, addrlen, as, sizeof(as));
5464                 verbose(VERB_ALGO, "cannot create tcp cp connection for "
5465                         "xfr %s to %s", zname, as);
5466                 return 0;
5467         }
5468         comm_timer_set(xfr->task_transfer->timer, &t);
5469         if(verbosity >= VERB_ALGO) {
5470                 char zname[255+1], as[256];
5471                 dname_str(xfr->name, zname);
5472                 addr_to_str(&addr, addrlen, as, sizeof(as));
5473                 verbose(VERB_ALGO, "auth zone %s transfer next %s fetch from %s started", zname,
5474                         (xfr->task_transfer->on_ixfr?"IXFR":"AXFR"), as);
5475         }
5476         return 1;
5477 }
5478
5479 /** perform next lookup, next transfer TCP, or end and resume wait time task */
5480 static void
5481 xfr_transfer_nexttarget_or_end(struct auth_xfer* xfr, struct module_env* env)
5482 {
5483         log_assert(xfr->task_transfer->worker == env->worker);
5484
5485         /* are we performing lookups? */
5486         while(xfr->task_transfer->lookup_target) {
5487                 if(xfr_transfer_lookup_host(xfr, env)) {
5488                         /* wait for lookup to finish,
5489                          * note that the hostname may be in unbound's cache
5490                          * and we may then get an instant cache response,
5491                          * and that calls the callback just like a full
5492                          * lookup and lookup failures also call callback */
5493                         if(verbosity >= VERB_ALGO) {
5494                                 char zname[255+1];
5495                                 dname_str(xfr->name, zname);
5496                                 verbose(VERB_ALGO, "auth zone %s transfer next target lookup", zname);
5497                         }
5498                         lock_basic_unlock(&xfr->lock);
5499                         return;
5500                 }
5501                 xfr_transfer_move_to_next_lookup(xfr, env);
5502         }
5503
5504         /* initiate TCP and fetch the zone from the master */
5505         /* and set timeout on it */
5506         while(!xfr_transfer_end_of_list(xfr)) {
5507                 xfr->task_transfer->master = xfr_transfer_current_master(xfr);
5508                 if(xfr_transfer_init_fetch(xfr, env)) {
5509                         /* successfully started, wait for callback */
5510                         lock_basic_unlock(&xfr->lock);
5511                         return;
5512                 }
5513                 /* failed to fetch, next master */
5514                 xfr_transfer_nextmaster(xfr);
5515         }
5516         if(verbosity >= VERB_ALGO) {
5517                 char zname[255+1];
5518                 dname_str(xfr->name, zname);
5519                 verbose(VERB_ALGO, "auth zone %s transfer failed, wait", zname);
5520         }
5521
5522         /* we failed to fetch the zone, move to wait task
5523          * use the shorter retry timeout */
5524         xfr_transfer_disown(xfr);
5525
5526         /* pick up the nextprobe task and wait */
5527         if(xfr->task_nextprobe->worker == NULL)
5528                 xfr_set_timeout(xfr, env, 1, 0);
5529         lock_basic_unlock(&xfr->lock);
5530 }
5531
5532 /** add addrs from A or AAAA rrset to the master */
5533 static void
5534 xfr_master_add_addrs(struct auth_master* m, struct ub_packed_rrset_key* rrset,
5535         uint16_t rrtype)
5536 {
5537         size_t i;
5538         struct packed_rrset_data* data;
5539         if(!m || !rrset) return;
5540         if(rrtype != LDNS_RR_TYPE_A && rrtype != LDNS_RR_TYPE_AAAA)
5541                 return;
5542         data = (struct packed_rrset_data*)rrset->entry.data;
5543         for(i=0; i<data->count; i++) {
5544                 struct auth_addr* a;
5545                 size_t len = data->rr_len[i] - 2;
5546                 uint8_t* rdata = data->rr_data[i]+2;
5547                 if(rrtype == LDNS_RR_TYPE_A && len != INET_SIZE)
5548                         continue; /* wrong length for A */
5549                 if(rrtype == LDNS_RR_TYPE_AAAA && len != INET6_SIZE)
5550                         continue; /* wrong length for AAAA */
5551
5552                 /* add and alloc it */
5553                 a = (struct auth_addr*)calloc(1, sizeof(*a));
5554                 if(!a) {
5555                         log_err("out of memory");
5556                         return;
5557                 }
5558                 if(rrtype == LDNS_RR_TYPE_A) {
5559                         struct sockaddr_in* sa;
5560                         a->addrlen = (socklen_t)sizeof(*sa);
5561                         sa = (struct sockaddr_in*)&a->addr;
5562                         sa->sin_family = AF_INET;
5563                         sa->sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5564                         memmove(&sa->sin_addr, rdata, INET_SIZE);
5565                 } else {
5566                         struct sockaddr_in6* sa;
5567                         a->addrlen = (socklen_t)sizeof(*sa);
5568                         sa = (struct sockaddr_in6*)&a->addr;
5569                         sa->sin6_family = AF_INET6;
5570                         sa->sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5571                         memmove(&sa->sin6_addr, rdata, INET6_SIZE);
5572                 }
5573                 if(verbosity >= VERB_ALGO) {
5574                         char s[64];
5575                         addr_to_str(&a->addr, a->addrlen, s, sizeof(s));
5576                         verbose(VERB_ALGO, "auth host %s lookup %s",
5577                                 m->host, s);
5578                 }
5579                 /* append to list */
5580                 a->next = m->list;
5581                 m->list = a;
5582         }
5583 }
5584
5585 /** callback for task_transfer lookup of host name, of A or AAAA */
5586 void auth_xfer_transfer_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
5587         enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
5588         int ATTR_UNUSED(was_ratelimited))
5589 {
5590         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5591         struct module_env* env;
5592         log_assert(xfr->task_transfer);
5593         lock_basic_lock(&xfr->lock);
5594         env = xfr->task_transfer->env;
5595         if(!env || env->outnet->want_to_quit) {
5596                 lock_basic_unlock(&xfr->lock);
5597                 return; /* stop on quit */
5598         }
5599
5600         /* process result */
5601         if(rcode == LDNS_RCODE_NOERROR) {
5602                 uint16_t wanted_qtype = LDNS_RR_TYPE_A;
5603                 struct regional* temp = env->scratch;
5604                 struct query_info rq;
5605                 struct reply_info* rep;
5606                 if(xfr->task_transfer->lookup_aaaa)
5607                         wanted_qtype = LDNS_RR_TYPE_AAAA;
5608                 memset(&rq, 0, sizeof(rq));
5609                 rep = parse_reply_in_temp_region(buf, temp, &rq);
5610                 if(rep && rq.qtype == wanted_qtype &&
5611                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
5612                         /* parsed successfully */
5613                         struct ub_packed_rrset_key* answer =
5614                                 reply_find_answer_rrset(&rq, rep);
5615                         if(answer) {
5616                                 xfr_master_add_addrs(xfr->task_transfer->
5617                                         lookup_target, answer, wanted_qtype);
5618                         } else {
5619                                 if(verbosity >= VERB_ALGO) {
5620                                         char zname[255+1];
5621                                         dname_str(xfr->name, zname);
5622                                         verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has nodata", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5623                                 }
5624                         }
5625                 } else {
5626                         if(verbosity >= VERB_ALGO) {
5627                                 char zname[255+1];
5628                                 dname_str(xfr->name, zname);
5629                                 verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has no answer", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5630                         }
5631                 }
5632                 regional_free_all(temp);
5633         } else {
5634                 if(verbosity >= VERB_ALGO) {
5635                         char zname[255+1];
5636                         dname_str(xfr->name, zname);
5637                         verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup failed", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5638                 }
5639         }
5640         if(xfr->task_transfer->lookup_target->list &&
5641                 xfr->task_transfer->lookup_target == xfr_transfer_current_master(xfr))
5642                 xfr->task_transfer->scan_addr = xfr->task_transfer->lookup_target->list;
5643
5644         /* move to lookup AAAA after A lookup, move to next hostname lookup,
5645          * or move to fetch the zone, or, if nothing to do, end task_transfer */
5646         xfr_transfer_move_to_next_lookup(xfr, env);
5647         xfr_transfer_nexttarget_or_end(xfr, env);
5648 }
5649
5650 /** check if xfer (AXFR or IXFR) packet is OK.
5651  * return false if we lost connection (SERVFAIL, or unreadable).
5652  * return false if we need to move from IXFR to AXFR, with gonextonfail
5653  *      set to false, so the same master is tried again, but with AXFR.
5654  * return true if fine to link into data.
5655  * return true with transferdone=true when the transfer has ended.
5656  */
5657 static int
5658 check_xfer_packet(sldns_buffer* pkt, struct auth_xfer* xfr,
5659         int* gonextonfail, int* transferdone)
5660 {
5661         uint8_t* wire = sldns_buffer_begin(pkt);
5662         int i;
5663         if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE) {
5664                 verbose(VERB_ALGO, "xfr to %s failed, packet too small",
5665                         xfr->task_transfer->master->host);
5666                 return 0;
5667         }
5668         if(!LDNS_QR_WIRE(wire)) {
5669                 verbose(VERB_ALGO, "xfr to %s failed, packet has no QR flag",
5670                         xfr->task_transfer->master->host);
5671                 return 0;
5672         }
5673         if(LDNS_TC_WIRE(wire)) {
5674                 verbose(VERB_ALGO, "xfr to %s failed, packet has TC flag",
5675                         xfr->task_transfer->master->host);
5676                 return 0;
5677         }
5678         /* check ID */
5679         if(LDNS_ID_WIRE(wire) != xfr->task_transfer->id) {
5680                 verbose(VERB_ALGO, "xfr to %s failed, packet wrong ID",
5681                         xfr->task_transfer->master->host);
5682                 return 0;
5683         }
5684         if(LDNS_RCODE_WIRE(wire) != LDNS_RCODE_NOERROR) {
5685                 char rcode[32];
5686                 sldns_wire2str_rcode_buf((int)LDNS_RCODE_WIRE(wire), rcode,
5687                         sizeof(rcode));
5688                 /* if we are doing IXFR, check for fallback */
5689                 if(xfr->task_transfer->on_ixfr) {
5690                         if(LDNS_RCODE_WIRE(wire) == LDNS_RCODE_NOTIMPL ||
5691                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_SERVFAIL ||
5692                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_REFUSED ||
5693                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_FORMERR) {
5694                                 verbose(VERB_ALGO, "xfr to %s, fallback "
5695                                         "from IXFR to AXFR (with rcode %s)",
5696                                         xfr->task_transfer->master->host,
5697                                         rcode);
5698                                 xfr->task_transfer->ixfr_fail = 1;
5699                                 *gonextonfail = 0;
5700                                 return 0;
5701                         }
5702                 }
5703                 verbose(VERB_ALGO, "xfr to %s failed, packet with rcode %s",
5704                         xfr->task_transfer->master->host, rcode);
5705                 return 0;
5706         }
5707         if(LDNS_OPCODE_WIRE(wire) != LDNS_PACKET_QUERY) {
5708                 verbose(VERB_ALGO, "xfr to %s failed, packet with bad opcode",
5709                         xfr->task_transfer->master->host);
5710                 return 0;
5711         }
5712         if(LDNS_QDCOUNT(wire) > 1) {
5713                 verbose(VERB_ALGO, "xfr to %s failed, packet has qdcount %d",
5714                         xfr->task_transfer->master->host,
5715                         (int)LDNS_QDCOUNT(wire));
5716                 return 0;
5717         }
5718
5719         /* check qname */
5720         sldns_buffer_set_position(pkt, LDNS_HEADER_SIZE);
5721         for(i=0; i<(int)LDNS_QDCOUNT(wire); i++) {
5722                 size_t pos = sldns_buffer_position(pkt);
5723                 uint16_t qtype, qclass;
5724                 if(pkt_dname_len(pkt) == 0) {
5725                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5726                                 "malformed dname",
5727                                 xfr->task_transfer->master->host);
5728                         return 0;
5729                 }
5730                 if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5731                         xfr->name) != 0) {
5732                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5733                                 "wrong qname",
5734                                 xfr->task_transfer->master->host);
5735                         return 0;
5736                 }
5737                 if(sldns_buffer_remaining(pkt) < 4) {
5738                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5739                                 "truncated query RR",
5740                                 xfr->task_transfer->master->host);
5741                         return 0;
5742                 }
5743                 qtype = sldns_buffer_read_u16(pkt);
5744                 qclass = sldns_buffer_read_u16(pkt);
5745                 if(qclass != xfr->dclass) {
5746                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5747                                 "wrong qclass",
5748                                 xfr->task_transfer->master->host);
5749                         return 0;
5750                 }
5751                 if(xfr->task_transfer->on_ixfr) {
5752                         if(qtype != LDNS_RR_TYPE_IXFR) {
5753                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5754                                         "with wrong qtype, expected IXFR",
5755                                 xfr->task_transfer->master->host);
5756                                 return 0;
5757                         }
5758                 } else {
5759                         if(qtype != LDNS_RR_TYPE_AXFR) {
5760                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5761                                         "with wrong qtype, expected AXFR",
5762                                 xfr->task_transfer->master->host);
5763                                 return 0;
5764                         }
5765                 }
5766         }
5767
5768         /* check parse of RRs in packet, store first SOA serial
5769          * to be able to detect last SOA (with that serial) to see if done */
5770         /* also check for IXFR 'zone up to date' reply */
5771         for(i=0; i<(int)LDNS_ANCOUNT(wire); i++) {
5772                 size_t pos = sldns_buffer_position(pkt);
5773                 uint16_t tp, rdlen;
5774                 if(pkt_dname_len(pkt) == 0) {
5775                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5776                                 "malformed dname in answer section",
5777                                 xfr->task_transfer->master->host);
5778                         return 0;
5779                 }
5780                 if(sldns_buffer_remaining(pkt) < 10) {
5781                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5782                                 "truncated RR",
5783                                 xfr->task_transfer->master->host);
5784                         return 0;
5785                 }
5786                 tp = sldns_buffer_read_u16(pkt);
5787                 (void)sldns_buffer_read_u16(pkt); /* class */
5788                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5789                 rdlen = sldns_buffer_read_u16(pkt);
5790                 if(sldns_buffer_remaining(pkt) < rdlen) {
5791                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5792                                 "truncated RR rdata",
5793                                 xfr->task_transfer->master->host);
5794                         return 0;
5795                 }
5796
5797                 /* RR parses (haven't checked rdata itself), now look at
5798                  * SOA records to see serial number */
5799                 if(xfr->task_transfer->rr_scan_num == 0 &&
5800                         tp != LDNS_RR_TYPE_SOA) {
5801                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5802                                 "malformed zone transfer, no start SOA",
5803                                 xfr->task_transfer->master->host);
5804                         return 0;
5805                 }
5806                 if(xfr->task_transfer->rr_scan_num == 1 &&
5807                         tp != LDNS_RR_TYPE_SOA) {
5808                         /* second RR is not a SOA record, this is not an IXFR
5809                          * the master is replying with an AXFR */
5810                         xfr->task_transfer->on_ixfr_is_axfr = 1;
5811                 }
5812                 if(tp == LDNS_RR_TYPE_SOA) {
5813                         uint32_t serial;
5814                         if(rdlen < 22) {
5815                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5816                                         "with SOA with malformed rdata",
5817                                         xfr->task_transfer->master->host);
5818                                 return 0;
5819                         }
5820                         if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5821                                 xfr->name) != 0) {
5822                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5823                                         "with SOA with wrong dname",
5824                                         xfr->task_transfer->master->host);
5825                                 return 0;
5826                         }
5827
5828                         /* read serial number of SOA */
5829                         serial = sldns_buffer_read_u32_at(pkt,
5830                                 sldns_buffer_position(pkt)+rdlen-20);
5831
5832                         /* check for IXFR 'zone has SOA x' reply */
5833                         if(xfr->task_transfer->on_ixfr &&
5834                                 xfr->task_transfer->rr_scan_num == 0 &&
5835                                 LDNS_ANCOUNT(wire)==1) {
5836                                 verbose(VERB_ALGO, "xfr to %s ended, "
5837                                         "IXFR reply that zone has serial %u,"
5838                                         " fallback from IXFR to AXFR",
5839                                         xfr->task_transfer->master->host,
5840                                         (unsigned)serial);
5841                                 xfr->task_transfer->ixfr_fail = 1;
5842                                 *gonextonfail = 0;
5843                                 return 0;
5844                         }
5845
5846                         /* if first SOA, store serial number */
5847                         if(xfr->task_transfer->got_xfr_serial == 0) {
5848                                 xfr->task_transfer->got_xfr_serial = 1;
5849                                 xfr->task_transfer->incoming_xfr_serial =
5850                                         serial;
5851                                 verbose(VERB_ALGO, "xfr %s: contains "
5852                                         "SOA serial %u",
5853                                         xfr->task_transfer->master->host,
5854                                         (unsigned)serial);
5855                         /* see if end of AXFR */
5856                         } else if(!xfr->task_transfer->on_ixfr ||
5857                                 xfr->task_transfer->on_ixfr_is_axfr) {
5858                                 /* second SOA with serial is the end
5859                                  * for AXFR */
5860                                 *transferdone = 1;
5861                                 verbose(VERB_ALGO, "xfr %s: last AXFR packet",
5862                                         xfr->task_transfer->master->host);
5863                         /* for IXFR, count SOA records with that serial */
5864                         } else if(xfr->task_transfer->incoming_xfr_serial ==
5865                                 serial && xfr->task_transfer->got_xfr_serial
5866                                 == 1) {
5867                                 xfr->task_transfer->got_xfr_serial++;
5868                         /* if not first soa, if serial==firstserial, the
5869                          * third time we are at the end, for IXFR */
5870                         } else if(xfr->task_transfer->incoming_xfr_serial ==
5871                                 serial && xfr->task_transfer->got_xfr_serial
5872                                 == 2) {
5873                                 verbose(VERB_ALGO, "xfr %s: last IXFR packet",
5874                                         xfr->task_transfer->master->host);
5875                                 *transferdone = 1;
5876                                 /* continue parse check, if that succeeds,
5877                                  * transfer is done */
5878                         }
5879                 }
5880                 xfr->task_transfer->rr_scan_num++;
5881
5882                 /* skip over RR rdata to go to the next RR */
5883                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5884         }
5885
5886         /* check authority section */
5887         /* we skip over the RRs checking packet format */
5888         for(i=0; i<(int)LDNS_NSCOUNT(wire); i++) {
5889                 uint16_t rdlen;
5890                 if(pkt_dname_len(pkt) == 0) {
5891                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5892                                 "malformed dname in authority section",
5893                                 xfr->task_transfer->master->host);
5894                         return 0;
5895                 }
5896                 if(sldns_buffer_remaining(pkt) < 10) {
5897                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5898                                 "truncated RR",
5899                                 xfr->task_transfer->master->host);
5900                         return 0;
5901                 }
5902                 (void)sldns_buffer_read_u16(pkt); /* type */
5903                 (void)sldns_buffer_read_u16(pkt); /* class */
5904                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5905                 rdlen = sldns_buffer_read_u16(pkt);
5906                 if(sldns_buffer_remaining(pkt) < rdlen) {
5907                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5908                                 "truncated RR rdata",
5909                                 xfr->task_transfer->master->host);
5910                         return 0;
5911                 }
5912                 /* skip over RR rdata to go to the next RR */
5913                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5914         }
5915
5916         /* check additional section */
5917         for(i=0; i<(int)LDNS_ARCOUNT(wire); i++) {
5918                 uint16_t rdlen;
5919                 if(pkt_dname_len(pkt) == 0) {
5920                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5921                                 "malformed dname in additional section",
5922                                 xfr->task_transfer->master->host);
5923                         return 0;
5924                 }
5925                 if(sldns_buffer_remaining(pkt) < 10) {
5926                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5927                                 "truncated RR",
5928                                 xfr->task_transfer->master->host);
5929                         return 0;
5930                 }
5931                 (void)sldns_buffer_read_u16(pkt); /* type */
5932                 (void)sldns_buffer_read_u16(pkt); /* class */
5933                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5934                 rdlen = sldns_buffer_read_u16(pkt);
5935                 if(sldns_buffer_remaining(pkt) < rdlen) {
5936                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5937                                 "truncated RR rdata",
5938                                 xfr->task_transfer->master->host);
5939                         return 0;
5940                 }
5941                 /* skip over RR rdata to go to the next RR */
5942                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5943         }
5944
5945         return 1;
5946 }
5947
5948 /** Link the data from this packet into the worklist of transferred data */
5949 static int
5950 xfer_link_data(sldns_buffer* pkt, struct auth_xfer* xfr)
5951 {
5952         /* alloc it */
5953         struct auth_chunk* e;
5954         e = (struct auth_chunk*)calloc(1, sizeof(*e));
5955         if(!e) return 0;
5956         e->next = NULL;
5957         e->len = sldns_buffer_limit(pkt);
5958         e->data = memdup(sldns_buffer_begin(pkt), e->len);
5959         if(!e->data) {
5960                 free(e);
5961                 return 0;
5962         }
5963
5964         /* alloc succeeded, link into list */
5965         if(!xfr->task_transfer->chunks_first)
5966                 xfr->task_transfer->chunks_first = e;
5967         if(xfr->task_transfer->chunks_last)
5968                 xfr->task_transfer->chunks_last->next = e;
5969         xfr->task_transfer->chunks_last = e;
5970         return 1;
5971 }
5972
5973 /** task transfer.  the list of data is complete. process it and if failed
5974  * move to next master, if succeeded, end the task transfer */
5975 static void
5976 process_list_end_transfer(struct auth_xfer* xfr, struct module_env* env)
5977 {
5978         int ixfr_fail = 0;
5979         if(xfr_process_chunk_list(xfr, env, &ixfr_fail)) {
5980                 /* it worked! */
5981                 auth_chunks_delete(xfr->task_transfer);
5982
5983                 /* we fetched the zone, move to wait task */
5984                 xfr_transfer_disown(xfr);
5985
5986                 if(xfr->notify_received && (!xfr->notify_has_serial ||
5987                         (xfr->notify_has_serial &&
5988                         xfr_serial_means_update(xfr, xfr->notify_serial)))) {
5989                         uint32_t sr = xfr->notify_serial;
5990                         int has_sr = xfr->notify_has_serial;
5991                         /* we received a notify while probe/transfer was
5992                          * in progress.  start a new probe and transfer */
5993                         xfr->notify_received = 0;
5994                         xfr->notify_has_serial = 0;
5995                         xfr->notify_serial = 0;
5996                         if(!xfr_start_probe(xfr, env, NULL)) {
5997                                 /* if we couldn't start it, already in
5998                                  * progress; restore notify serial,
5999                                  * while xfr still locked */
6000                                 xfr->notify_received = 1;
6001                                 xfr->notify_has_serial = has_sr;
6002                                 xfr->notify_serial = sr;
6003                                 lock_basic_unlock(&xfr->lock);
6004                         }
6005                         return;
6006                 } else {
6007                         /* pick up the nextprobe task and wait (normail wait time) */
6008                         if(xfr->task_nextprobe->worker == NULL)
6009                                 xfr_set_timeout(xfr, env, 0, 0);
6010                 }
6011                 lock_basic_unlock(&xfr->lock);
6012                 return;
6013         }
6014         /* processing failed */
6015         /* when done, delete data from list */
6016         auth_chunks_delete(xfr->task_transfer);
6017         if(ixfr_fail) {
6018                 xfr->task_transfer->ixfr_fail = 1;
6019         } else {
6020                 xfr_transfer_nextmaster(xfr);
6021         }
6022         xfr_transfer_nexttarget_or_end(xfr, env);
6023 }
6024
6025 /** callback for the task_transfer timer */
6026 void
6027 auth_xfer_transfer_timer_callback(void* arg)
6028 {
6029         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6030         struct module_env* env;
6031         int gonextonfail = 1;
6032         log_assert(xfr->task_transfer);
6033         lock_basic_lock(&xfr->lock);
6034         env = xfr->task_transfer->env;
6035         if(!env || env->outnet->want_to_quit) {
6036                 lock_basic_unlock(&xfr->lock);
6037                 return; /* stop on quit */
6038         }
6039
6040         verbose(VERB_ALGO, "xfr stopped, connection timeout to %s",
6041                 xfr->task_transfer->master->host);
6042
6043         /* see if IXFR caused the failure, if so, try AXFR */
6044         if(xfr->task_transfer->on_ixfr) {
6045                 xfr->task_transfer->ixfr_possible_timeout_count++;
6046                 if(xfr->task_transfer->ixfr_possible_timeout_count >=
6047                         NUM_TIMEOUTS_FALLBACK_IXFR) {
6048                         verbose(VERB_ALGO, "xfr to %s, fallback "
6049                                 "from IXFR to AXFR (because of timeouts)",
6050                                 xfr->task_transfer->master->host);
6051                         xfr->task_transfer->ixfr_fail = 1;
6052                         gonextonfail = 0;
6053                 }
6054         }
6055
6056         /* delete transferred data from list */
6057         auth_chunks_delete(xfr->task_transfer);
6058         comm_point_delete(xfr->task_transfer->cp);
6059         xfr->task_transfer->cp = NULL;
6060         if(gonextonfail)
6061                 xfr_transfer_nextmaster(xfr);
6062         xfr_transfer_nexttarget_or_end(xfr, env);
6063 }
6064
6065 /** callback for task_transfer tcp connections */
6066 int
6067 auth_xfer_transfer_tcp_callback(struct comm_point* c, void* arg, int err,
6068         struct comm_reply* ATTR_UNUSED(repinfo))
6069 {
6070         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6071         struct module_env* env;
6072         int gonextonfail = 1;
6073         int transferdone = 0;
6074         log_assert(xfr->task_transfer);
6075         lock_basic_lock(&xfr->lock);
6076         env = xfr->task_transfer->env;
6077         if(!env || env->outnet->want_to_quit) {
6078                 lock_basic_unlock(&xfr->lock);
6079                 return 0; /* stop on quit */
6080         }
6081         /* stop the timer */
6082         comm_timer_disable(xfr->task_transfer->timer);
6083
6084         if(err != NETEVENT_NOERROR) {
6085                 /* connection failed, closed, or timeout */
6086                 /* stop this transfer, cleanup
6087                  * and continue task_transfer*/
6088                 verbose(VERB_ALGO, "xfr stopped, connection lost to %s",
6089                         xfr->task_transfer->master->host);
6090
6091                 /* see if IXFR caused the failure, if so, try AXFR */
6092                 if(xfr->task_transfer->on_ixfr) {
6093                         xfr->task_transfer->ixfr_possible_timeout_count++;
6094                         if(xfr->task_transfer->ixfr_possible_timeout_count >=
6095                                 NUM_TIMEOUTS_FALLBACK_IXFR) {
6096                                 verbose(VERB_ALGO, "xfr to %s, fallback "
6097                                         "from IXFR to AXFR (because of timeouts)",
6098                                         xfr->task_transfer->master->host);
6099                                 xfr->task_transfer->ixfr_fail = 1;
6100                                 gonextonfail = 0;
6101                         }
6102                 }
6103
6104         failed:
6105                 /* delete transferred data from list */
6106                 auth_chunks_delete(xfr->task_transfer);
6107                 comm_point_delete(xfr->task_transfer->cp);
6108                 xfr->task_transfer->cp = NULL;
6109                 if(gonextonfail)
6110                         xfr_transfer_nextmaster(xfr);
6111                 xfr_transfer_nexttarget_or_end(xfr, env);
6112                 return 0;
6113         }
6114         /* note that IXFR worked without timeout */
6115         if(xfr->task_transfer->on_ixfr)
6116                 xfr->task_transfer->ixfr_possible_timeout_count = 0;
6117
6118         /* handle returned packet */
6119         /* if it fails, cleanup and end this transfer */
6120         /* if it needs to fallback from IXFR to AXFR, do that */
6121         if(!check_xfer_packet(c->buffer, xfr, &gonextonfail, &transferdone)) {
6122                 goto failed;
6123         }
6124         /* if it is good, link it into the list of data */
6125         /* if the link into list of data fails (malloc fail) cleanup and end */
6126         if(!xfer_link_data(c->buffer, xfr)) {
6127                 verbose(VERB_ALGO, "xfr stopped to %s, malloc failed",
6128                         xfr->task_transfer->master->host);
6129                 goto failed;
6130         }
6131         /* if the transfer is done now, disconnect and process the list */
6132         if(transferdone) {
6133                 comm_point_delete(xfr->task_transfer->cp);
6134                 xfr->task_transfer->cp = NULL;
6135                 process_list_end_transfer(xfr, env);
6136                 return 0;
6137         }
6138
6139         /* if we want to read more messages, setup the commpoint to read
6140          * a DNS packet, and the timeout */
6141         lock_basic_unlock(&xfr->lock);
6142         c->tcp_is_reading = 1;
6143         sldns_buffer_clear(c->buffer);
6144         comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
6145         return 0;
6146 }
6147
6148 /** callback for task_transfer http connections */
6149 int
6150 auth_xfer_transfer_http_callback(struct comm_point* c, void* arg, int err,
6151         struct comm_reply* repinfo)
6152 {
6153         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6154         struct module_env* env;
6155         log_assert(xfr->task_transfer);
6156         lock_basic_lock(&xfr->lock);
6157         env = xfr->task_transfer->env;
6158         if(!env || env->outnet->want_to_quit) {
6159                 lock_basic_unlock(&xfr->lock);
6160                 return 0; /* stop on quit */
6161         }
6162         verbose(VERB_ALGO, "auth zone transfer http callback");
6163         /* stop the timer */
6164         comm_timer_disable(xfr->task_transfer->timer);
6165
6166         if(err != NETEVENT_NOERROR && err != NETEVENT_DONE) {
6167                 /* connection failed, closed, or timeout */
6168                 /* stop this transfer, cleanup
6169                  * and continue task_transfer*/
6170                 verbose(VERB_ALGO, "http stopped, connection lost to %s",
6171                         xfr->task_transfer->master->host);
6172         failed:
6173                 /* delete transferred data from list */
6174                 auth_chunks_delete(xfr->task_transfer);
6175                 if(repinfo) repinfo->c = NULL; /* signal cp deleted to
6176                                 the routine calling this callback */
6177                 comm_point_delete(xfr->task_transfer->cp);
6178                 xfr->task_transfer->cp = NULL;
6179                 xfr_transfer_nextmaster(xfr);
6180                 xfr_transfer_nexttarget_or_end(xfr, env);
6181                 return 0;
6182         }
6183
6184         /* if it is good, link it into the list of data */
6185         /* if the link into list of data fails (malloc fail) cleanup and end */
6186         if(sldns_buffer_limit(c->buffer) > 0) {
6187                 verbose(VERB_ALGO, "auth zone http queued up %d bytes",
6188                         (int)sldns_buffer_limit(c->buffer));
6189                 if(!xfer_link_data(c->buffer, xfr)) {
6190                         verbose(VERB_ALGO, "http stopped to %s, malloc failed",
6191                                 xfr->task_transfer->master->host);
6192                         goto failed;
6193                 }
6194         }
6195         /* if the transfer is done now, disconnect and process the list */
6196         if(err == NETEVENT_DONE) {
6197                 if(repinfo) repinfo->c = NULL; /* signal cp deleted to
6198                                 the routine calling this callback */
6199                 comm_point_delete(xfr->task_transfer->cp);
6200                 xfr->task_transfer->cp = NULL;
6201                 process_list_end_transfer(xfr, env);
6202                 return 0;
6203         }
6204
6205         /* if we want to read more messages, setup the commpoint to read
6206          * a DNS packet, and the timeout */
6207         lock_basic_unlock(&xfr->lock);
6208         c->tcp_is_reading = 1;
6209         sldns_buffer_clear(c->buffer);
6210         comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
6211         return 0;
6212 }
6213
6214
6215 /** start transfer task by this worker , xfr is locked. */
6216 static void
6217 xfr_start_transfer(struct auth_xfer* xfr, struct module_env* env,
6218         struct auth_master* master)
6219 {
6220         log_assert(xfr->task_transfer != NULL);
6221         log_assert(xfr->task_transfer->worker == NULL);
6222         log_assert(xfr->task_transfer->chunks_first == NULL);
6223         log_assert(xfr->task_transfer->chunks_last == NULL);
6224         xfr->task_transfer->worker = env->worker;
6225         xfr->task_transfer->env = env;
6226
6227         /* init transfer process */
6228         /* find that master in the transfer's list of masters? */
6229         xfr_transfer_start_list(xfr, master);
6230         /* start lookup for hostnames in transfer master list */
6231         xfr_transfer_start_lookups(xfr);
6232
6233         /* initiate TCP, and set timeout on it */
6234         xfr_transfer_nexttarget_or_end(xfr, env);
6235 }
6236
6237 /** disown task_probe.  caller must hold xfr.lock */
6238 static void
6239 xfr_probe_disown(struct auth_xfer* xfr)
6240 {
6241         /* remove timer (from this worker's event base) */
6242         comm_timer_delete(xfr->task_probe->timer);
6243         xfr->task_probe->timer = NULL;
6244         /* remove the commpoint */
6245         comm_point_delete(xfr->task_probe->cp);
6246         xfr->task_probe->cp = NULL;
6247         /* we don't own this item anymore */
6248         xfr->task_probe->worker = NULL;
6249         xfr->task_probe->env = NULL;
6250 }
6251
6252 /** send the UDP probe to the master, this is part of task_probe */
6253 static int
6254 xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
6255         int timeout)
6256 {
6257         struct sockaddr_storage addr;
6258         socklen_t addrlen = 0;
6259         struct timeval t;
6260         /* pick master */
6261         struct auth_master* master = xfr_probe_current_master(xfr);
6262         char *auth_name = NULL;
6263         if(!master) return 0;
6264         if(master->allow_notify) return 0; /* only for notify */
6265         if(master->http) return 0; /* only masters get SOA UDP probe,
6266                 not urls, if those are in this list */
6267
6268         /* get master addr */
6269         if(xfr->task_probe->scan_addr) {
6270                 addrlen = xfr->task_probe->scan_addr->addrlen;
6271                 memmove(&addr, &xfr->task_probe->scan_addr->addr, addrlen);
6272         } else {
6273                 if(!authextstrtoaddr(master->host, &addr, &addrlen, &auth_name)) {
6274                         /* the ones that are not in addr format are supposed
6275                          * to be looked up.  The lookup has failed however,
6276                          * so skip them */
6277                         char zname[255+1];
6278                         dname_str(xfr->name, zname);
6279                         log_err("%s: failed lookup, cannot probe to master %s",
6280                                 zname, master->host);
6281                         return 0;
6282                 }
6283                 if (auth_name != NULL) {
6284                         if (addr.ss_family == AF_INET
6285                         &&  (int)ntohs(((struct sockaddr_in *)&addr)->sin_port)
6286                             == env->cfg->ssl_port)
6287                                 ((struct sockaddr_in *)&addr)->sin_port
6288                                         = htons((uint16_t)env->cfg->port);
6289                         else if (addr.ss_family == AF_INET6
6290                         &&  (int)ntohs(((struct sockaddr_in6 *)&addr)->sin6_port)
6291                             == env->cfg->ssl_port)
6292                                 ((struct sockaddr_in6 *)&addr)->sin6_port
6293                                         = htons((uint16_t)env->cfg->port);
6294                 }
6295         }
6296
6297         /* create packet */
6298         /* create new ID for new probes, but not on timeout retries,
6299          * this means we'll accept replies to previous retries to same ip */
6300         if(timeout == AUTH_PROBE_TIMEOUT)
6301                 xfr->task_probe->id = GET_RANDOM_ID(env->rnd);
6302         xfr_create_soa_probe_packet(xfr, env->scratch_buffer,
6303                 xfr->task_probe->id);
6304         /* we need to remove the cp if we have a different ip4/ip6 type now */
6305         if(xfr->task_probe->cp &&
6306                 ((xfr->task_probe->cp_is_ip6 && !addr_is_ip6(&addr, addrlen)) ||
6307                 (!xfr->task_probe->cp_is_ip6 && addr_is_ip6(&addr, addrlen)))
6308                 ) {
6309                 comm_point_delete(xfr->task_probe->cp);
6310                 xfr->task_probe->cp = NULL;
6311         }
6312         if(!xfr->task_probe->cp) {
6313                 if(addr_is_ip6(&addr, addrlen))
6314                         xfr->task_probe->cp_is_ip6 = 1;
6315                 else    xfr->task_probe->cp_is_ip6 = 0;
6316                 xfr->task_probe->cp = outnet_comm_point_for_udp(env->outnet,
6317                         auth_xfer_probe_udp_callback, xfr, &addr, addrlen);
6318                 if(!xfr->task_probe->cp) {
6319                         char zname[255+1], as[256];
6320                         dname_str(xfr->name, zname);
6321                         addr_to_str(&addr, addrlen, as, sizeof(as));
6322                         verbose(VERB_ALGO, "cannot create udp cp for "
6323                                 "probe %s to %s", zname, as);
6324                         return 0;
6325                 }
6326         }
6327         if(!xfr->task_probe->timer) {
6328                 xfr->task_probe->timer = comm_timer_create(env->worker_base,
6329                         auth_xfer_probe_timer_callback, xfr);
6330                 if(!xfr->task_probe->timer) {
6331                         log_err("malloc failure");
6332                         return 0;
6333                 }
6334         }
6335
6336         /* send udp packet */
6337         if(!comm_point_send_udp_msg(xfr->task_probe->cp, env->scratch_buffer,
6338                 (struct sockaddr*)&addr, addrlen, 0)) {
6339                 char zname[255+1], as[256];
6340                 dname_str(xfr->name, zname);
6341                 addr_to_str(&addr, addrlen, as, sizeof(as));
6342                 verbose(VERB_ALGO, "failed to send soa probe for %s to %s",
6343                         zname, as);
6344                 return 0;
6345         }
6346         if(verbosity >= VERB_ALGO) {
6347                 char zname[255+1], as[256];
6348                 dname_str(xfr->name, zname);
6349                 addr_to_str(&addr, addrlen, as, sizeof(as));
6350                 verbose(VERB_ALGO, "auth zone %s soa probe sent to %s", zname,
6351                         as);
6352         }
6353         xfr->task_probe->timeout = timeout;
6354 #ifndef S_SPLINT_S
6355         t.tv_sec = timeout/1000;
6356         t.tv_usec = (timeout%1000)*1000;
6357 #endif
6358         comm_timer_set(xfr->task_probe->timer, &t);
6359
6360         return 1;
6361 }
6362
6363 /** callback for task_probe timer */
6364 void
6365 auth_xfer_probe_timer_callback(void* arg)
6366 {
6367         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6368         struct module_env* env;
6369         log_assert(xfr->task_probe);
6370         lock_basic_lock(&xfr->lock);
6371         env = xfr->task_probe->env;
6372         if(!env || env->outnet->want_to_quit) {
6373                 lock_basic_unlock(&xfr->lock);
6374                 return; /* stop on quit */
6375         }
6376
6377         if(verbosity >= VERB_ALGO) {
6378                 char zname[255+1];
6379                 dname_str(xfr->name, zname);
6380                 verbose(VERB_ALGO, "auth zone %s soa probe timeout", zname);
6381         }
6382         if(xfr->task_probe->timeout <= AUTH_PROBE_TIMEOUT_STOP) {
6383                 /* try again with bigger timeout */
6384                 if(xfr_probe_send_probe(xfr, env, xfr->task_probe->timeout*2)) {
6385                         lock_basic_unlock(&xfr->lock);
6386                         return;
6387                 }
6388         }
6389         /* delete commpoint so a new one is created, with a fresh port nr */
6390         comm_point_delete(xfr->task_probe->cp);
6391         xfr->task_probe->cp = NULL;
6392
6393         /* too many timeouts (or fail to send), move to next or end */
6394         xfr_probe_nextmaster(xfr);
6395         xfr_probe_send_or_end(xfr, env);
6396 }
6397
6398 /** callback for task_probe udp packets */
6399 int
6400 auth_xfer_probe_udp_callback(struct comm_point* c, void* arg, int err,
6401         struct comm_reply* repinfo)
6402 {
6403         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6404         struct module_env* env;
6405         log_assert(xfr->task_probe);
6406         lock_basic_lock(&xfr->lock);
6407         env = xfr->task_probe->env;
6408         if(!env || env->outnet->want_to_quit) {
6409                 lock_basic_unlock(&xfr->lock);
6410                 return 0; /* stop on quit */
6411         }
6412
6413         /* the comm_point_udp_callback is in a for loop for NUM_UDP_PER_SELECT
6414          * and we set rep.c=NULL to stop if from looking inside the commpoint*/
6415         repinfo->c = NULL;
6416         /* stop the timer */
6417         comm_timer_disable(xfr->task_probe->timer);
6418
6419         /* see if we got a packet and what that means */
6420         if(err == NETEVENT_NOERROR) {
6421                 uint32_t serial = 0;
6422                 if(check_packet_ok(c->buffer, LDNS_RR_TYPE_SOA, xfr,
6423                         &serial)) {
6424                         /* successful lookup */
6425                         if(verbosity >= VERB_ALGO) {
6426                                 char buf[256];
6427                                 dname_str(xfr->name, buf);
6428                                 verbose(VERB_ALGO, "auth zone %s: soa probe "
6429                                         "serial is %u", buf, (unsigned)serial);
6430                         }
6431                         /* see if this serial indicates that the zone has
6432                          * to be updated */
6433                         if(xfr_serial_means_update(xfr, serial)) {
6434                                 /* if updated, start the transfer task, if needed */
6435                                 verbose(VERB_ALGO, "auth_zone updated, start transfer");
6436                                 if(xfr->task_transfer->worker == NULL) {
6437                                         struct auth_master* master =
6438                                                 xfr_probe_current_master(xfr);
6439                                         /* if we have download URLs use them
6440                                          * in preference to this master we
6441                                          * just probed the SOA from */
6442                                         if(xfr->task_transfer->masters &&
6443                                                 xfr->task_transfer->masters->http)
6444                                                 master = NULL;
6445                                         xfr_probe_disown(xfr);
6446                                         xfr_start_transfer(xfr, env, master);
6447                                         return 0;
6448
6449                                 }
6450                                 /* other tasks are running, we don't do this anymore */
6451                                 xfr_probe_disown(xfr);
6452                                 lock_basic_unlock(&xfr->lock);
6453                                 /* return, we don't sent a reply to this udp packet,
6454                                  * and we setup the tasks to do next */
6455                                 return 0;
6456                         } else {
6457                                 verbose(VERB_ALGO, "auth_zone master reports unchanged soa serial");
6458                                 /* we if cannot find updates amongst the
6459                                  * masters, this means we then have a new lease
6460                                  * on the zone */
6461                                 xfr->task_probe->have_new_lease = 1;
6462                         }
6463                 } else {
6464                         if(verbosity >= VERB_ALGO) {
6465                                 char buf[256];
6466                                 dname_str(xfr->name, buf);
6467                                 verbose(VERB_ALGO, "auth zone %s: bad reply to soa probe", buf);
6468                         }
6469                 }
6470         } else {
6471                 if(verbosity >= VERB_ALGO) {
6472                         char buf[256];
6473                         dname_str(xfr->name, buf);
6474                         verbose(VERB_ALGO, "auth zone %s: soa probe failed", buf);
6475                 }
6476         }
6477
6478         /* failed lookup or not an update */
6479         /* delete commpoint so a new one is created, with a fresh port nr */
6480         comm_point_delete(xfr->task_probe->cp);
6481         xfr->task_probe->cp = NULL;
6482
6483         /* if the result was not a successfull probe, we need
6484          * to send the next one */
6485         xfr_probe_nextmaster(xfr);
6486         xfr_probe_send_or_end(xfr, env);
6487         return 0;
6488 }
6489
6490 /** lookup a host name for its addresses, if needed */
6491 static int
6492 xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
6493 {
6494         struct sockaddr_storage addr;
6495         socklen_t addrlen = 0;
6496         struct auth_master* master = xfr->task_probe->lookup_target;
6497         struct query_info qinfo;
6498         uint16_t qflags = BIT_RD;
6499         uint8_t dname[LDNS_MAX_DOMAINLEN+1];
6500         struct edns_data edns;
6501         sldns_buffer* buf = env->scratch_buffer;
6502         if(!master) return 0;
6503         if(extstrtoaddr(master->host, &addr, &addrlen)) {
6504                 /* not needed, host is in IP addr format */
6505                 return 0;
6506         }
6507         if(master->allow_notify && !master->http &&
6508                 strchr(master->host, '/') != NULL &&
6509                 strchr(master->host, '/') == strrchr(master->host, '/')) {
6510                 return 0; /* is IP/prefix format, not something to look up */
6511         }
6512
6513         /* use mesh_new_callback to probe for non-addr hosts,
6514          * and then wait for them to be looked up (in cache, or query) */
6515         qinfo.qname_len = sizeof(dname);
6516         if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
6517                 != 0) {
6518                 log_err("cannot parse host name of master %s", master->host);
6519                 return 0;
6520         }
6521         qinfo.qname = dname;
6522         qinfo.qclass = xfr->dclass;
6523         qinfo.qtype = LDNS_RR_TYPE_A;
6524         if(xfr->task_probe->lookup_aaaa)
6525                 qinfo.qtype = LDNS_RR_TYPE_AAAA;
6526         qinfo.local_alias = NULL;
6527         if(verbosity >= VERB_ALGO) {
6528                 char buf1[512];
6529                 char buf2[LDNS_MAX_DOMAINLEN+1];
6530                 dname_str(xfr->name, buf2);
6531                 snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
6532                         " for task_probe", buf2);
6533                 log_query_info(VERB_ALGO, buf1, &qinfo);
6534         }
6535         edns.edns_present = 1;
6536         edns.ext_rcode = 0;
6537         edns.edns_version = 0;
6538         edns.bits = EDNS_DO;
6539         edns.opt_list = NULL;
6540         edns.padding_block_size = 0;
6541         if(sldns_buffer_capacity(buf) < 65535)
6542                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
6543         else    edns.udp_size = 65535;
6544
6545         /* unlock xfr during mesh_new_callback() because the callback can be
6546          * called straight away */
6547         lock_basic_unlock(&xfr->lock);
6548         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
6549                 &auth_xfer_probe_lookup_callback, xfr)) {
6550                 lock_basic_lock(&xfr->lock);
6551                 log_err("out of memory lookup up master %s", master->host);
6552                 return 0;
6553         }
6554         lock_basic_lock(&xfr->lock);
6555         return 1;
6556 }
6557
6558 /** move to sending the probe packets, next if fails. task_probe */
6559 static void
6560 xfr_probe_send_or_end(struct auth_xfer* xfr, struct module_env* env)
6561 {
6562         /* are we doing hostname lookups? */
6563         while(xfr->task_probe->lookup_target) {
6564                 if(xfr_probe_lookup_host(xfr, env)) {
6565                         /* wait for lookup to finish,
6566                          * note that the hostname may be in unbound's cache
6567                          * and we may then get an instant cache response,
6568                          * and that calls the callback just like a full
6569                          * lookup and lookup failures also call callback */
6570                         if(verbosity >= VERB_ALGO) {
6571                                 char zname[255+1];
6572                                 dname_str(xfr->name, zname);
6573                                 verbose(VERB_ALGO, "auth zone %s probe next target lookup", zname);
6574                         }
6575                         lock_basic_unlock(&xfr->lock);
6576                         return;
6577                 }
6578                 xfr_probe_move_to_next_lookup(xfr, env);
6579         }
6580         /* probe of list has ended.  Create or refresh the list of of
6581          * allow_notify addrs */
6582         probe_copy_masters_for_allow_notify(xfr);
6583         if(verbosity >= VERB_ALGO) {
6584                 char zname[255+1];
6585                 dname_str(xfr->name, zname);
6586                 verbose(VERB_ALGO, "auth zone %s probe: notify addrs updated", zname);
6587         }
6588         if(xfr->task_probe->only_lookup) {
6589                 /* only wanted lookups for copy, stop probe and start wait */
6590                 xfr->task_probe->only_lookup = 0;
6591                 if(verbosity >= VERB_ALGO) {
6592                         char zname[255+1];
6593                         dname_str(xfr->name, zname);
6594                         verbose(VERB_ALGO, "auth zone %s probe: finished only_lookup", zname);
6595                 }
6596                 xfr_probe_disown(xfr);
6597                 if(xfr->task_nextprobe->worker == NULL)
6598                         xfr_set_timeout(xfr, env, 0, 0);
6599                 lock_basic_unlock(&xfr->lock);
6600                 return;
6601         }
6602
6603         /* send probe packets */
6604         while(!xfr_probe_end_of_list(xfr)) {
6605                 if(xfr_probe_send_probe(xfr, env, AUTH_PROBE_TIMEOUT)) {
6606                         /* successfully sent probe, wait for callback */
6607                         lock_basic_unlock(&xfr->lock);
6608                         return;
6609                 }
6610                 /* failed to send probe, next master */
6611                 xfr_probe_nextmaster(xfr);
6612         }
6613
6614         /* done with probe sequence, wait */
6615         if(xfr->task_probe->have_new_lease) {
6616                 /* if zone not updated, start the wait timer again */
6617                 if(verbosity >= VERB_ALGO) {
6618                         char zname[255+1];
6619                         dname_str(xfr->name, zname);
6620                         verbose(VERB_ALGO, "auth_zone %s unchanged, new lease, wait", zname);
6621                 }
6622                 xfr_probe_disown(xfr);
6623                 if(xfr->have_zone)
6624                         xfr->lease_time = *env->now;
6625                 if(xfr->task_nextprobe->worker == NULL)
6626                         xfr_set_timeout(xfr, env, 0, 0);
6627         } else {
6628                 if(verbosity >= VERB_ALGO) {
6629                         char zname[255+1];
6630                         dname_str(xfr->name, zname);
6631                         verbose(VERB_ALGO, "auth zone %s soa probe failed, wait to retry", zname);
6632                 }
6633                 /* we failed to send this as well, move to the wait task,
6634                  * use the shorter retry timeout */
6635                 xfr_probe_disown(xfr);
6636                 /* pick up the nextprobe task and wait */
6637                 if(xfr->task_nextprobe->worker == NULL)
6638                         xfr_set_timeout(xfr, env, 1, 0);
6639         }
6640
6641         lock_basic_unlock(&xfr->lock);
6642 }
6643
6644 /** callback for task_probe lookup of host name, of A or AAAA */
6645 void auth_xfer_probe_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
6646         enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
6647         int ATTR_UNUSED(was_ratelimited))
6648 {
6649         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6650         struct module_env* env;
6651         log_assert(xfr->task_probe);
6652         lock_basic_lock(&xfr->lock);
6653         env = xfr->task_probe->env;
6654         if(!env || env->outnet->want_to_quit) {
6655                 lock_basic_unlock(&xfr->lock);
6656                 return; /* stop on quit */
6657         }
6658
6659         /* process result */
6660         if(rcode == LDNS_RCODE_NOERROR) {
6661                 uint16_t wanted_qtype = LDNS_RR_TYPE_A;
6662                 struct regional* temp = env->scratch;
6663                 struct query_info rq;
6664                 struct reply_info* rep;
6665                 if(xfr->task_probe->lookup_aaaa)
6666                         wanted_qtype = LDNS_RR_TYPE_AAAA;
6667                 memset(&rq, 0, sizeof(rq));
6668                 rep = parse_reply_in_temp_region(buf, temp, &rq);
6669                 if(rep && rq.qtype == wanted_qtype &&
6670                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
6671                         /* parsed successfully */
6672                         struct ub_packed_rrset_key* answer =
6673                                 reply_find_answer_rrset(&rq, rep);
6674                         if(answer) {
6675                                 xfr_master_add_addrs(xfr->task_probe->
6676                                         lookup_target, answer, wanted_qtype);
6677                         } else {
6678                                 if(verbosity >= VERB_ALGO) {
6679                                         char zname[255+1];
6680                                         dname_str(xfr->name, zname);
6681                                         verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has nodata", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6682                                 }
6683                         }
6684                 } else {
6685                         if(verbosity >= VERB_ALGO) {
6686                                 char zname[255+1];
6687                                 dname_str(xfr->name, zname);
6688                                 verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has no address", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6689                         }
6690                 }
6691                 regional_free_all(temp);
6692         } else {
6693                 if(verbosity >= VERB_ALGO) {
6694                         char zname[255+1];
6695                         dname_str(xfr->name, zname);
6696                         verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup failed", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6697                 }
6698         }
6699         if(xfr->task_probe->lookup_target->list &&
6700                 xfr->task_probe->lookup_target == xfr_probe_current_master(xfr))
6701                 xfr->task_probe->scan_addr = xfr->task_probe->lookup_target->list;
6702
6703         /* move to lookup AAAA after A lookup, move to next hostname lookup,
6704          * or move to send the probes, or, if nothing to do, end task_probe */
6705         xfr_probe_move_to_next_lookup(xfr, env);
6706         xfr_probe_send_or_end(xfr, env);
6707 }
6708
6709 /** disown task_nextprobe.  caller must hold xfr.lock */
6710 static void
6711 xfr_nextprobe_disown(struct auth_xfer* xfr)
6712 {
6713         /* delete the timer, because the next worker to pick this up may
6714          * not have the same event base */
6715         comm_timer_delete(xfr->task_nextprobe->timer);
6716         xfr->task_nextprobe->timer = NULL;
6717         xfr->task_nextprobe->next_probe = 0;
6718         /* we don't own this item anymore */
6719         xfr->task_nextprobe->worker = NULL;
6720         xfr->task_nextprobe->env = NULL;
6721 }
6722
6723 /** xfer nextprobe timeout callback, this is part of task_nextprobe */
6724 void
6725 auth_xfer_timer(void* arg)
6726 {
6727         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6728         struct module_env* env;
6729         log_assert(xfr->task_nextprobe);
6730         lock_basic_lock(&xfr->lock);
6731         env = xfr->task_nextprobe->env;
6732         if(!env || env->outnet->want_to_quit) {
6733                 lock_basic_unlock(&xfr->lock);
6734                 return; /* stop on quit */
6735         }
6736
6737         /* see if zone has expired, and if so, also set auth_zone expired */
6738         if(xfr->have_zone && !xfr->zone_expired &&
6739            *env->now >= xfr->lease_time + xfr->expiry) {
6740                 lock_basic_unlock(&xfr->lock);
6741                 auth_xfer_set_expired(xfr, env, 1);
6742                 lock_basic_lock(&xfr->lock);
6743         }
6744
6745         xfr_nextprobe_disown(xfr);
6746
6747         if(!xfr_start_probe(xfr, env, NULL)) {
6748                 /* not started because already in progress */
6749                 lock_basic_unlock(&xfr->lock);
6750         }
6751 }
6752
6753 /** return true if there are probe (SOA UDP query) targets in the master list*/
6754 static int
6755 have_probe_targets(struct auth_master* list)
6756 {
6757         struct auth_master* p;
6758         for(p=list; p; p = p->next) {
6759                 if(!p->allow_notify && p->host)
6760                         return 1;
6761         }
6762         return 0;
6763 }
6764
6765 /** start task_probe if possible, if no masters for probe start task_transfer
6766  * returns true if task has been started, and false if the task is already
6767  * in progress. */
6768 static int
6769 xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
6770         struct auth_master* spec)
6771 {
6772         /* see if we need to start a probe (or maybe it is already in
6773          * progress (due to notify)) */
6774         if(xfr->task_probe->worker == NULL) {
6775                 if(!have_probe_targets(xfr->task_probe->masters) &&
6776                         !(xfr->task_probe->only_lookup &&
6777                         xfr->task_probe->masters != NULL)) {
6778                         /* useless to pick up task_probe, no masters to
6779                          * probe. Instead attempt to pick up task transfer */
6780                         if(xfr->task_transfer->worker == NULL) {
6781                                 xfr_start_transfer(xfr, env, spec);
6782                                 return 1;
6783                         }
6784                         /* task transfer already in progress */
6785                         return 0;
6786                 }
6787
6788                 /* pick up the probe task ourselves */
6789                 xfr->task_probe->worker = env->worker;
6790                 xfr->task_probe->env = env;
6791                 xfr->task_probe->cp = NULL;
6792
6793                 /* start the task */
6794                 /* have not seen a new lease yet, this scan */
6795                 xfr->task_probe->have_new_lease = 0;
6796                 /* if this was a timeout, no specific first master to scan */
6797                 /* otherwise, spec is nonNULL the notified master, scan
6798                  * first and also transfer first from it */
6799                 xfr_probe_start_list(xfr, spec);
6800                 /* setup to start the lookup of hostnames of masters afresh */
6801                 xfr_probe_start_lookups(xfr);
6802                 /* send the probe packet or next send, or end task */
6803                 xfr_probe_send_or_end(xfr, env);
6804                 return 1;
6805         }
6806         return 0;
6807 }
6808
6809 /** for task_nextprobe.
6810  * determine next timeout for auth_xfer. Also (re)sets timer.
6811  * @param xfr: task structure
6812  * @param env: module environment, with worker and time.
6813  * @param failure: set true if timer should be set for failure retry.
6814  * @param lookup_only: only perform lookups when timer done, 0 sec timeout
6815  */
6816 static void
6817 xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
6818         int failure, int lookup_only)
6819 {
6820         struct timeval tv;
6821         log_assert(xfr->task_nextprobe != NULL);
6822         log_assert(xfr->task_nextprobe->worker == NULL ||
6823                 xfr->task_nextprobe->worker == env->worker);
6824         /* normally, nextprobe = startoflease + refresh,
6825          * but if expiry is sooner, use that one.
6826          * after a failure, use the retry timer instead. */
6827         xfr->task_nextprobe->next_probe = *env->now;
6828         if(xfr->lease_time && !failure)
6829                 xfr->task_nextprobe->next_probe = xfr->lease_time;
6830
6831         if(!failure) {
6832                 xfr->task_nextprobe->backoff = 0;
6833         } else {
6834                 if(xfr->task_nextprobe->backoff == 0)
6835                                 xfr->task_nextprobe->backoff = 3;
6836                 else    xfr->task_nextprobe->backoff *= 2;
6837                 if(xfr->task_nextprobe->backoff > AUTH_TRANSFER_MAX_BACKOFF)
6838                         xfr->task_nextprobe->backoff =
6839                                 AUTH_TRANSFER_MAX_BACKOFF;
6840         }
6841
6842         if(xfr->have_zone) {
6843                 time_t wait = xfr->refresh;
6844                 if(failure) wait = xfr->retry;
6845                 if(xfr->expiry < wait)
6846                         xfr->task_nextprobe->next_probe += xfr->expiry;
6847                 else    xfr->task_nextprobe->next_probe += wait;
6848                 if(failure)
6849                         xfr->task_nextprobe->next_probe +=
6850                                 xfr->task_nextprobe->backoff;
6851                 /* put the timer exactly on expiry, if possible */
6852                 if(xfr->lease_time && xfr->lease_time+xfr->expiry <
6853                         xfr->task_nextprobe->next_probe &&
6854                         xfr->lease_time+xfr->expiry > *env->now)
6855                         xfr->task_nextprobe->next_probe =
6856                                 xfr->lease_time+xfr->expiry;
6857         } else {
6858                 xfr->task_nextprobe->next_probe +=
6859                         xfr->task_nextprobe->backoff;
6860         }
6861
6862         if(!xfr->task_nextprobe->timer) {
6863                 xfr->task_nextprobe->timer = comm_timer_create(
6864                         env->worker_base, auth_xfer_timer, xfr);
6865                 if(!xfr->task_nextprobe->timer) {
6866                         /* failed to malloc memory. likely zone transfer
6867                          * also fails for that. skip the timeout */
6868                         char zname[255+1];
6869                         dname_str(xfr->name, zname);
6870                         log_err("cannot allocate timer, no refresh for %s",
6871                                 zname);
6872                         return;
6873                 }
6874         }
6875         xfr->task_nextprobe->worker = env->worker;
6876         xfr->task_nextprobe->env = env;
6877         if(*(xfr->task_nextprobe->env->now) <= xfr->task_nextprobe->next_probe)
6878                 tv.tv_sec = xfr->task_nextprobe->next_probe -
6879                         *(xfr->task_nextprobe->env->now);
6880         else    tv.tv_sec = 0;
6881         if(tv.tv_sec != 0 && lookup_only && xfr->task_probe->masters) {
6882                 /* don't lookup_only, if lookup timeout is 0 anyway,
6883                  * or if we don't have masters to lookup */
6884                 tv.tv_sec = 0;
6885                 if(xfr->task_probe->worker == NULL)
6886                         xfr->task_probe->only_lookup = 1;
6887         }
6888         if(verbosity >= VERB_ALGO) {
6889                 char zname[255+1];
6890                 dname_str(xfr->name, zname);
6891                 verbose(VERB_ALGO, "auth zone %s timeout in %d seconds",
6892                         zname, (int)tv.tv_sec);
6893         }
6894         tv.tv_usec = 0;
6895         comm_timer_set(xfr->task_nextprobe->timer, &tv);
6896 }
6897
6898 /** initial pick up of worker timeouts, ties events to worker event loop */
6899 void
6900 auth_xfer_pickup_initial(struct auth_zones* az, struct module_env* env)
6901 {
6902         struct auth_xfer* x;
6903         lock_rw_wrlock(&az->lock);
6904         RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6905                 lock_basic_lock(&x->lock);
6906                 /* set lease_time, because we now have timestamp in env,
6907                  * (not earlier during startup and apply_cfg), and this
6908                  * notes the start time when the data was acquired */
6909                 if(x->have_zone)
6910                         x->lease_time = *env->now;
6911                 if(x->task_nextprobe && x->task_nextprobe->worker == NULL) {
6912                         xfr_set_timeout(x, env, 0, 1);
6913                 }
6914                 lock_basic_unlock(&x->lock);
6915         }
6916         lock_rw_unlock(&az->lock);
6917 }
6918
6919 void auth_zones_cleanup(struct auth_zones* az)
6920 {
6921         struct auth_xfer* x;
6922         lock_rw_wrlock(&az->lock);
6923         RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6924                 lock_basic_lock(&x->lock);
6925                 if(x->task_nextprobe && x->task_nextprobe->worker != NULL) {
6926                         xfr_nextprobe_disown(x);
6927                 }
6928                 if(x->task_probe && x->task_probe->worker != NULL) {
6929                         xfr_probe_disown(x);
6930                 }
6931                 if(x->task_transfer && x->task_transfer->worker != NULL) {
6932                         auth_chunks_delete(x->task_transfer);
6933                         xfr_transfer_disown(x);
6934                 }
6935                 lock_basic_unlock(&x->lock);
6936         }
6937         lock_rw_unlock(&az->lock);
6938 }
6939
6940 /**
6941  * malloc the xfer and tasks
6942  * @param z: auth_zone with name of zone.
6943  */
6944 static struct auth_xfer*
6945 auth_xfer_new(struct auth_zone* z)
6946 {
6947         struct auth_xfer* xfr;
6948         xfr = (struct auth_xfer*)calloc(1, sizeof(*xfr));
6949         if(!xfr) return NULL;
6950         xfr->name = memdup(z->name, z->namelen);
6951         if(!xfr->name) {
6952                 free(xfr);
6953                 return NULL;
6954         }
6955         xfr->node.key = xfr;
6956         xfr->namelen = z->namelen;
6957         xfr->namelabs = z->namelabs;
6958         xfr->dclass = z->dclass;
6959
6960         xfr->task_nextprobe = (struct auth_nextprobe*)calloc(1,
6961                 sizeof(struct auth_nextprobe));
6962         if(!xfr->task_nextprobe) {
6963                 free(xfr->name);
6964                 free(xfr);
6965                 return NULL;
6966         }
6967         xfr->task_probe = (struct auth_probe*)calloc(1,
6968                 sizeof(struct auth_probe));
6969         if(!xfr->task_probe) {
6970                 free(xfr->task_nextprobe);
6971                 free(xfr->name);
6972                 free(xfr);
6973                 return NULL;
6974         }
6975         xfr->task_transfer = (struct auth_transfer*)calloc(1,
6976                 sizeof(struct auth_transfer));
6977         if(!xfr->task_transfer) {
6978                 free(xfr->task_probe);
6979                 free(xfr->task_nextprobe);
6980                 free(xfr->name);
6981                 free(xfr);
6982                 return NULL;
6983         }
6984
6985         lock_basic_init(&xfr->lock);
6986         lock_protect(&xfr->lock, &xfr->name, sizeof(xfr->name));
6987         lock_protect(&xfr->lock, &xfr->namelen, sizeof(xfr->namelen));
6988         lock_protect(&xfr->lock, xfr->name, xfr->namelen);
6989         lock_protect(&xfr->lock, &xfr->namelabs, sizeof(xfr->namelabs));
6990         lock_protect(&xfr->lock, &xfr->dclass, sizeof(xfr->dclass));
6991         lock_protect(&xfr->lock, &xfr->notify_received, sizeof(xfr->notify_received));
6992         lock_protect(&xfr->lock, &xfr->notify_serial, sizeof(xfr->notify_serial));
6993         lock_protect(&xfr->lock, &xfr->zone_expired, sizeof(xfr->zone_expired));
6994         lock_protect(&xfr->lock, &xfr->have_zone, sizeof(xfr->have_zone));
6995         lock_protect(&xfr->lock, &xfr->serial, sizeof(xfr->serial));
6996         lock_protect(&xfr->lock, &xfr->retry, sizeof(xfr->retry));
6997         lock_protect(&xfr->lock, &xfr->refresh, sizeof(xfr->refresh));
6998         lock_protect(&xfr->lock, &xfr->expiry, sizeof(xfr->expiry));
6999         lock_protect(&xfr->lock, &xfr->lease_time, sizeof(xfr->lease_time));
7000         lock_protect(&xfr->lock, &xfr->task_nextprobe->worker,
7001                 sizeof(xfr->task_nextprobe->worker));
7002         lock_protect(&xfr->lock, &xfr->task_probe->worker,
7003                 sizeof(xfr->task_probe->worker));
7004         lock_protect(&xfr->lock, &xfr->task_transfer->worker,
7005                 sizeof(xfr->task_transfer->worker));
7006         lock_basic_lock(&xfr->lock);
7007         return xfr;
7008 }
7009
7010 /** Create auth_xfer structure.
7011  * This populates the have_zone, soa values, and so on times.
7012  * and sets the timeout, if a zone transfer is needed a short timeout is set.
7013  * For that the auth_zone itself must exist (and read in zonefile)
7014  * returns false on alloc failure. */
7015 struct auth_xfer*
7016 auth_xfer_create(struct auth_zones* az, struct auth_zone* z)
7017 {
7018         struct auth_xfer* xfr;
7019
7020         /* malloc it */
7021         xfr = auth_xfer_new(z);
7022         if(!xfr) {
7023                 log_err("malloc failure");
7024                 return NULL;
7025         }
7026         /* insert in tree */
7027         (void)rbtree_insert(&az->xtree, &xfr->node);
7028         return xfr;
7029 }
7030
7031 /** create new auth_master structure */
7032 static struct auth_master*
7033 auth_master_new(struct auth_master*** list)
7034 {
7035         struct auth_master *m;
7036         m = (struct auth_master*)calloc(1, sizeof(*m));
7037         if(!m) {
7038                 log_err("malloc failure");
7039                 return NULL;
7040         }
7041         /* set first pointer to m, or next pointer of previous element to m */
7042         (**list) = m;
7043         /* store m's next pointer as future point to store at */
7044         (*list) = &(m->next);
7045         return m;
7046 }
7047
7048 /** dup_prefix : create string from initial part of other string, malloced */
7049 static char*
7050 dup_prefix(char* str, size_t num)
7051 {
7052         char* result;
7053         size_t len = strlen(str);
7054         if(len < num) num = len; /* not more than strlen */
7055         result = (char*)malloc(num+1);
7056         if(!result) {
7057                 log_err("malloc failure");
7058                 return result;
7059         }
7060         memmove(result, str, num);
7061         result[num] = 0;
7062         return result;
7063 }
7064
7065 /** dup string and print error on error */
7066 static char*
7067 dup_all(char* str)
7068 {
7069         char* result = strdup(str);
7070         if(!result) {
7071                 log_err("malloc failure");
7072                 return NULL;
7073         }
7074         return result;
7075 }
7076
7077 /** find first of two characters */
7078 static char*
7079 str_find_first_of_chars(char* s, char a, char b)
7080 {
7081         char* ra = strchr(s, a);
7082         char* rb = strchr(s, b);
7083         if(!ra) return rb;
7084         if(!rb) return ra;
7085         if(ra < rb) return ra;
7086         return rb;
7087 }
7088
7089 /** parse URL into host and file parts, false on malloc or parse error */
7090 static int
7091 parse_url(char* url, char** host, char** file, int* port, int* ssl)
7092 {
7093         char* p = url;
7094         /* parse http://www.example.com/file.htm
7095          * or http://127.0.0.1   (index.html)
7096          * or https://[::1@1234]/a/b/c/d */
7097         *ssl = 1;
7098         *port = AUTH_HTTPS_PORT;
7099
7100         /* parse http:// or https:// */
7101         if(strncmp(p, "http://", 7) == 0) {
7102                 p += 7;
7103                 *ssl = 0;
7104                 *port = AUTH_HTTP_PORT;
7105         } else if(strncmp(p, "https://", 8) == 0) {
7106                 p += 8;
7107         } else if(strstr(p, "://") && strchr(p, '/') > strstr(p, "://") &&
7108                 strchr(p, ':') >= strstr(p, "://")) {
7109                 char* uri = dup_prefix(p, (size_t)(strstr(p, "://")-p));
7110                 log_err("protocol %s:// not supported (for url %s)",
7111                         uri?uri:"", p);
7112                 free(uri);
7113                 return 0;
7114         }
7115
7116         /* parse hostname part */
7117         if(p[0] == '[') {
7118                 char* end = strchr(p, ']');
7119                 p++; /* skip over [ */
7120                 if(end) {
7121                         *host = dup_prefix(p, (size_t)(end-p));
7122                         if(!*host) return 0;
7123                         p = end+1; /* skip over ] */
7124                 } else {
7125                         *host = dup_all(p);
7126                         if(!*host) return 0;
7127                         p = end;
7128                 }
7129         } else {
7130                 char* end = str_find_first_of_chars(p, ':', '/');
7131                 if(end) {
7132                         *host = dup_prefix(p, (size_t)(end-p));
7133                         if(!*host) return 0;
7134                 } else {
7135                         *host = dup_all(p);
7136                         if(!*host) return 0;
7137                 }
7138                 p = end; /* at next : or / or NULL */
7139         }
7140
7141         /* parse port number */
7142         if(p && p[0] == ':') {
7143                 char* end = NULL;
7144                 *port = strtol(p+1, &end, 10);
7145                 p = end;
7146         }
7147
7148         /* parse filename part */
7149         while(p && *p == '/')
7150                 p++;
7151         if(!p || p[0] == 0)
7152                 *file = strdup("index.html");
7153         else    *file = strdup(p);
7154         if(!*file) {
7155                 log_err("malloc failure");
7156                 return 0;
7157         }
7158         return 1;
7159 }
7160
7161 int
7162 xfer_set_masters(struct auth_master** list, struct config_auth* c,
7163         int with_http)
7164 {
7165         struct auth_master* m;
7166         struct config_strlist* p;
7167         /* list points to the first, or next pointer for the new element */
7168         while(*list) {
7169                 list = &( (*list)->next );
7170         }
7171         if(with_http)
7172           for(p = c->urls; p; p = p->next) {
7173                 m = auth_master_new(&list);
7174                 if(!m) return 0;
7175                 m->http = 1;
7176                 if(!parse_url(p->str, &m->host, &m->file, &m->port, &m->ssl))
7177                         return 0;
7178         }
7179         for(p = c->masters; p; p = p->next) {
7180                 m = auth_master_new(&list);
7181                 if(!m) return 0;
7182                 m->ixfr = 1; /* this flag is not configurable */
7183                 m->host = strdup(p->str);
7184                 if(!m->host) {
7185                         log_err("malloc failure");
7186                         return 0;
7187                 }
7188         }
7189         for(p = c->allow_notify; p; p = p->next) {
7190                 m = auth_master_new(&list);
7191                 if(!m) return 0;
7192                 m->allow_notify = 1;
7193                 m->host = strdup(p->str);
7194                 if(!m->host) {
7195                         log_err("malloc failure");
7196                         return 0;
7197                 }
7198         }
7199         return 1;
7200 }
7201
7202 #define SERIAL_BITS     32
7203 int
7204 compare_serial(uint32_t a, uint32_t b)
7205 {
7206         const uint32_t cutoff = ((uint32_t) 1 << (SERIAL_BITS - 1));
7207
7208         if (a == b) {
7209                 return 0;
7210         } else if ((a < b && b - a < cutoff) || (a > b && a - b > cutoff)) {
7211                 return -1;
7212         } else {
7213                 return 1;
7214         }
7215 }
7216
7217 int zonemd_hashalgo_supported(int hashalgo)
7218 {
7219         if(hashalgo == ZONEMD_ALGO_SHA384) return 1;
7220         if(hashalgo == ZONEMD_ALGO_SHA512) return 1;
7221         return 0;
7222 }
7223
7224 int zonemd_scheme_supported(int scheme)
7225 {
7226         if(scheme == ZONEMD_SCHEME_SIMPLE) return 1;
7227         return 0;
7228 }
7229
7230 /** initialize hash for hashing with zonemd hash algo */
7231 static struct secalgo_hash* zonemd_digest_init(int hashalgo, char** reason)
7232 {
7233         struct secalgo_hash *h;
7234         if(hashalgo == ZONEMD_ALGO_SHA384) {
7235                 /* sha384 */
7236                 h = secalgo_hash_create_sha384();
7237                 if(!h)
7238                         *reason = "digest sha384 could not be created";
7239                 return h;
7240         } else if(hashalgo == ZONEMD_ALGO_SHA512) {
7241                 /* sha512 */
7242                 h = secalgo_hash_create_sha512();
7243                 if(!h)
7244                         *reason = "digest sha512 could not be created";
7245                 return h;
7246         }
7247         /* unknown hash algo */
7248         *reason = "unsupported algorithm";
7249         return NULL;
7250 }
7251
7252 /** update the hash for zonemd */
7253 static int zonemd_digest_update(int hashalgo, struct secalgo_hash* h,
7254         uint8_t* data, size_t len, char** reason)
7255 {
7256         if(hashalgo == ZONEMD_ALGO_SHA384) {
7257                 if(!secalgo_hash_update(h, data, len)) {
7258                         *reason = "digest sha384 failed";
7259                         return 0;
7260                 }
7261                 return 1;
7262         } else if(hashalgo == ZONEMD_ALGO_SHA512) {
7263                 if(!secalgo_hash_update(h, data, len)) {
7264                         *reason = "digest sha512 failed";
7265                         return 0;
7266                 }
7267                 return 1;
7268         }
7269         /* unknown hash algo */
7270         *reason = "unsupported algorithm";
7271         return 0;
7272 }
7273
7274 /** finish the hash for zonemd */
7275 static int zonemd_digest_finish(int hashalgo, struct secalgo_hash* h,
7276         uint8_t* result, size_t hashlen, size_t* resultlen, char** reason)
7277 {
7278         if(hashalgo == ZONEMD_ALGO_SHA384) {
7279                 if(hashlen < 384/8) {
7280                         *reason = "digest buffer too small for sha384";
7281                         return 0;
7282                 }
7283                 if(!secalgo_hash_final(h, result, hashlen, resultlen)) {
7284                         *reason = "digest sha384 finish failed";
7285                         return 0;
7286                 }
7287                 return 1;
7288         } else if(hashalgo == ZONEMD_ALGO_SHA512) {
7289                 if(hashlen < 512/8) {
7290                         *reason = "digest buffer too small for sha512";
7291                         return 0;
7292                 }
7293                 if(!secalgo_hash_final(h, result, hashlen, resultlen)) {
7294                         *reason = "digest sha512 finish failed";
7295                         return 0;
7296                 }
7297                 return 1;
7298         }
7299         /* unknown algo */
7300         *reason = "unsupported algorithm";
7301         return 0;
7302 }
7303
7304 /** add rrsets from node to the list */
7305 static size_t authdata_rrsets_to_list(struct auth_rrset** array,
7306         size_t arraysize, struct auth_rrset* first)
7307 {
7308         struct auth_rrset* rrset = first;
7309         size_t num = 0;
7310         while(rrset) {
7311                 if(num >= arraysize)
7312                         return num;
7313                 array[num] = rrset;
7314                 num++;
7315                 rrset = rrset->next;
7316         }
7317         return num;
7318 }
7319
7320 /** compare rr list entries */
7321 static int rrlist_compare(const void* arg1, const void* arg2)
7322 {
7323         struct auth_rrset* r1 = *(struct auth_rrset**)arg1;
7324         struct auth_rrset* r2 = *(struct auth_rrset**)arg2;
7325         uint16_t t1, t2;
7326         if(r1 == NULL) t1 = LDNS_RR_TYPE_RRSIG;
7327         else t1 = r1->type;
7328         if(r2 == NULL) t2 = LDNS_RR_TYPE_RRSIG;
7329         else t2 = r2->type;
7330         if(t1 < t2)
7331                 return -1;
7332         if(t1 > t2)
7333                 return 1;
7334         return 0;
7335 }
7336
7337 /** add type RRSIG to rr list if not one there already,
7338  * this is to perform RRSIG collate processing at that point. */
7339 static void addrrsigtype_if_needed(struct auth_rrset** array,
7340         size_t arraysize, size_t* rrnum, struct auth_data* node)
7341 {
7342         if(az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
7343                 return; /* already one there */
7344         if((*rrnum) >= arraysize)
7345                 return; /* array too small? */
7346         array[*rrnum] = NULL; /* nothing there, but need entry in list */
7347         (*rrnum)++;
7348 }
7349
7350 /** collate the RRs in an RRset using the simple scheme */
7351 static int zonemd_simple_rrset(struct auth_zone* z, int hashalgo,
7352         struct secalgo_hash* h, struct auth_data* node,
7353         struct auth_rrset* rrset, struct regional* region,
7354         struct sldns_buffer* buf, char** reason)
7355 {
7356         /* canonicalize */
7357         struct ub_packed_rrset_key key;
7358         memset(&key, 0, sizeof(key));
7359         key.entry.key = &key;
7360         key.entry.data = rrset->data;
7361         key.rk.dname = node->name;
7362         key.rk.dname_len = node->namelen;
7363         key.rk.type = htons(rrset->type);
7364         key.rk.rrset_class = htons(z->dclass);
7365         if(!rrset_canonicalize_to_buffer(region, buf, &key)) {
7366                 *reason = "out of memory";
7367                 return 0;
7368         }
7369         regional_free_all(region);
7370
7371         /* hash */
7372         if(!zonemd_digest_update(hashalgo, h, sldns_buffer_begin(buf),
7373                 sldns_buffer_limit(buf), reason)) {
7374                 return 0;
7375         }
7376         return 1;
7377 }
7378
7379 /** count number of RRSIGs in a domain name rrset list */
7380 static size_t zonemd_simple_count_rrsig(struct auth_rrset* rrset,
7381         struct auth_rrset** rrlist, size_t rrnum,
7382         struct auth_zone* z, struct auth_data* node)
7383 {
7384         size_t i, count = 0;
7385         if(rrset) {
7386                 size_t j;
7387                 for(j = 0; j<rrset->data->count; j++) {
7388                         if(rrsig_rdata_get_type_covered(rrset->data->
7389                                 rr_data[j], rrset->data->rr_len[j]) ==
7390                                 LDNS_RR_TYPE_ZONEMD &&
7391                                 query_dname_compare(z->name, node->name)==0) {
7392                                 /* omit RRSIGs over type ZONEMD at apex */
7393                                 continue;
7394                         }
7395                         count++;
7396                 }
7397         }
7398         for(i=0; i<rrnum; i++) {
7399                 if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7400                         query_dname_compare(z->name, node->name)==0) {
7401                         /* omit RRSIGs over type ZONEMD at apex */
7402                         continue;
7403                 }
7404                 count += (rrlist[i]?rrlist[i]->data->rrsig_count:0);
7405         }
7406         return count;
7407 }
7408
7409 /** allocate sparse rrset data for the number of entries in tepm region */
7410 static int zonemd_simple_rrsig_allocs(struct regional* region,
7411         struct packed_rrset_data* data, size_t count)
7412 {
7413         data->rr_len = regional_alloc(region, sizeof(*data->rr_len) * count);
7414         if(!data->rr_len) {
7415                 return 0;
7416         }
7417         data->rr_ttl = regional_alloc(region, sizeof(*data->rr_ttl) * count);
7418         if(!data->rr_ttl) {
7419                 return 0;
7420         }
7421         data->rr_data = regional_alloc(region, sizeof(*data->rr_data) * count);
7422         if(!data->rr_data) {
7423                 return 0;
7424         }
7425         return 1;
7426 }
7427
7428 /** add the RRSIGs from the rrs in the domain into the data */
7429 static void add_rrlist_rrsigs_into_data(struct packed_rrset_data* data,
7430         size_t* done, struct auth_rrset** rrlist, size_t rrnum,
7431         struct auth_zone* z, struct auth_data* node)
7432 {
7433         size_t i;
7434         for(i=0; i<rrnum; i++) {
7435                 size_t j;
7436                 if(!rrlist[i])
7437                         continue;
7438                 if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7439                         query_dname_compare(z->name, node->name)==0) {
7440                         /* omit RRSIGs over type ZONEMD at apex */
7441                         continue;
7442                 }
7443                 for(j = 0; j<rrlist[i]->data->rrsig_count; j++) {
7444                         data->rr_len[*done] = rrlist[i]->data->rr_len[rrlist[i]->data->count + j];
7445                         data->rr_ttl[*done] = rrlist[i]->data->rr_ttl[rrlist[i]->data->count + j];
7446                         /* reference the rdata in the rrset, no need to
7447                          * copy it, it is no longer needed at the end of
7448                          * the routine */
7449                         data->rr_data[*done] = rrlist[i]->data->rr_data[rrlist[i]->data->count + j];
7450                         (*done)++;
7451                 }
7452         }
7453 }
7454
7455 static void add_rrset_into_data(struct packed_rrset_data* data,
7456         size_t* done, struct auth_rrset* rrset,
7457         struct auth_zone* z, struct auth_data* node)
7458 {
7459         if(rrset) {
7460                 size_t j;
7461                 for(j = 0; j<rrset->data->count; j++) {
7462                         if(rrsig_rdata_get_type_covered(rrset->data->
7463                                 rr_data[j], rrset->data->rr_len[j]) ==
7464                                 LDNS_RR_TYPE_ZONEMD &&
7465                                 query_dname_compare(z->name, node->name)==0) {
7466                                 /* omit RRSIGs over type ZONEMD at apex */
7467                                 continue;
7468                         }
7469                         data->rr_len[*done] = rrset->data->rr_len[j];
7470                         data->rr_ttl[*done] = rrset->data->rr_ttl[j];
7471                         /* reference the rdata in the rrset, no need to
7472                          * copy it, it is no longer need at the end of
7473                          * the routine */
7474                         data->rr_data[*done] = rrset->data->rr_data[j];
7475                         (*done)++;
7476                 }
7477         }
7478 }
7479
7480 /** collate the RRSIGs using the simple scheme */
7481 static int zonemd_simple_rrsig(struct auth_zone* z, int hashalgo,
7482         struct secalgo_hash* h, struct auth_data* node,
7483         struct auth_rrset* rrset, struct auth_rrset** rrlist, size_t rrnum,
7484         struct regional* region, struct sldns_buffer* buf, char** reason)
7485 {
7486         /* the rrset pointer can be NULL, this means it is type RRSIG and
7487          * there is no ordinary type RRSIG there.  The RRSIGs are stored
7488          * with the RRsets in their data.
7489          *
7490          * The RRset pointer can be nonNULL. This happens if there is
7491          * no RR that is covered by the RRSIG for the domain.  Then this
7492          * RRSIG RR is stored in an rrset of type RRSIG. The other RRSIGs
7493          * are stored in the rrset entries for the RRs in the rr list for
7494          * the domain node.  We need to collate the rrset's data, if any, and
7495          * the rrlist's rrsigs */
7496         /* if this is the apex, omit RRSIGs that cover type ZONEMD */
7497         /* build rrsig rrset */
7498         size_t done = 0;
7499         struct ub_packed_rrset_key key;
7500         struct packed_rrset_data data;
7501         memset(&key, 0, sizeof(key));
7502         memset(&data, 0, sizeof(data));
7503         key.entry.key = &key;
7504         key.entry.data = &data;
7505         key.rk.dname = node->name;
7506         key.rk.dname_len = node->namelen;
7507         key.rk.type = htons(LDNS_RR_TYPE_RRSIG);
7508         key.rk.rrset_class = htons(z->dclass);
7509         data.count = zonemd_simple_count_rrsig(rrset, rrlist, rrnum, z, node);
7510         if(!zonemd_simple_rrsig_allocs(region, &data, data.count)) {
7511                 *reason = "out of memory";
7512                 regional_free_all(region);
7513                 return 0;
7514         }
7515         /* all the RRSIGs stored in the other rrsets for this domain node */
7516         add_rrlist_rrsigs_into_data(&data, &done, rrlist, rrnum, z, node);
7517         /* plus the RRSIGs stored in an rrset of type RRSIG for this node */
7518         add_rrset_into_data(&data, &done, rrset, z, node);
7519
7520         /* canonicalize */
7521         if(!rrset_canonicalize_to_buffer(region, buf, &key)) {
7522                 *reason = "out of memory";
7523                 regional_free_all(region);
7524                 return 0;
7525         }
7526         regional_free_all(region);
7527
7528         /* hash */
7529         if(!zonemd_digest_update(hashalgo, h, sldns_buffer_begin(buf),
7530                 sldns_buffer_limit(buf), reason)) {
7531                 return 0;
7532         }
7533         return 1;
7534 }
7535
7536 /** collate a domain's rrsets using the simple scheme */
7537 static int zonemd_simple_domain(struct auth_zone* z, int hashalgo,
7538         struct secalgo_hash* h, struct auth_data* node,
7539         struct regional* region, struct sldns_buffer* buf, char** reason)
7540 {
7541         const size_t rrlistsize = 65536;
7542         struct auth_rrset* rrlist[rrlistsize];
7543         size_t i, rrnum = 0;
7544         /* see if the domain is out of scope, the zone origin,
7545          * that would be omitted */
7546         if(!dname_subdomain_c(node->name, z->name))
7547                 return 1; /* continue */
7548         /* loop over the rrsets in ascending order. */
7549         rrnum = authdata_rrsets_to_list(rrlist, rrlistsize, node->rrsets);
7550         addrrsigtype_if_needed(rrlist, rrlistsize, &rrnum, node);
7551         qsort(rrlist, rrnum, sizeof(*rrlist), rrlist_compare);
7552         for(i=0; i<rrnum; i++) {
7553                 if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7554                         query_dname_compare(z->name, node->name) == 0) {
7555                         /* omit type ZONEMD at apex */
7556                         continue;
7557                 }
7558                 if(rrlist[i] == NULL || rrlist[i]->type ==
7559                         LDNS_RR_TYPE_RRSIG) {
7560                         if(!zonemd_simple_rrsig(z, hashalgo, h, node,
7561                                 rrlist[i], rrlist, rrnum, region, buf, reason))
7562                                 return 0;
7563                 } else if(!zonemd_simple_rrset(z, hashalgo, h, node,
7564                         rrlist[i], region, buf, reason)) {
7565                         return 0;
7566                 }
7567         }
7568         return 1;
7569 }
7570
7571 /** collate the zone using the simple scheme */
7572 static int zonemd_simple_collate(struct auth_zone* z, int hashalgo,
7573         struct secalgo_hash* h, struct regional* region,
7574         struct sldns_buffer* buf, char** reason)
7575 {
7576         /* our tree is sorted in canonical order, so we can just loop over
7577          * the tree */
7578         struct auth_data* n;
7579         RBTREE_FOR(n, struct auth_data*, &z->data) {
7580                 if(!zonemd_simple_domain(z, hashalgo, h, n, region, buf,
7581                         reason))
7582                         return 0;
7583         }
7584         return 1;
7585 }
7586
7587 int auth_zone_generate_zonemd_hash(struct auth_zone* z, int scheme,
7588         int hashalgo, uint8_t* hash, size_t hashlen, size_t* resultlen,
7589         struct regional* region, struct sldns_buffer* buf, char** reason)
7590 {
7591         struct secalgo_hash* h = zonemd_digest_init(hashalgo, reason);
7592         if(!h) {
7593                 if(!*reason)
7594                         *reason = "digest init fail";
7595                 return 0;
7596         }
7597         if(scheme == ZONEMD_SCHEME_SIMPLE) {
7598                 if(!zonemd_simple_collate(z, hashalgo, h, region, buf, reason)) {
7599                         if(!*reason) *reason = "scheme simple collate fail";
7600                         secalgo_hash_delete(h);
7601                         return 0;
7602                 }
7603         }
7604         if(!zonemd_digest_finish(hashalgo, h, hash, hashlen, resultlen,
7605                 reason)) {
7606                 secalgo_hash_delete(h);
7607                 *reason = "digest finish fail";
7608                 return 0;
7609         }
7610         secalgo_hash_delete(h);
7611         return 1;
7612 }
7613
7614 int auth_zone_generate_zonemd_check(struct auth_zone* z, int scheme,
7615         int hashalgo, uint8_t* hash, size_t hashlen, struct regional* region,
7616         struct sldns_buffer* buf, char** reason)
7617 {
7618         uint8_t gen[512];
7619         size_t genlen = 0;
7620         if(!zonemd_hashalgo_supported(hashalgo)) {
7621                 *reason = "unsupported algorithm";
7622                 return 0;
7623         }
7624         if(!zonemd_scheme_supported(scheme)) {
7625                 *reason = "unsupported scheme";
7626                 return 0;
7627         }
7628         if(hashlen < 12) {
7629                 /* the ZONEMD draft requires digests to fail if too small */
7630                 *reason = "digest length too small, less than 12";
7631                 return 0;
7632         }
7633         /* generate digest */
7634         if(!auth_zone_generate_zonemd_hash(z, scheme, hashalgo, gen,
7635                 sizeof(gen), &genlen, region, buf, reason)) {
7636                 /* reason filled in by zonemd hash routine */
7637                 return 0;
7638         }
7639         /* check digest length */
7640         if(hashlen != genlen) {
7641                 *reason = "incorrect digest length";
7642                 if(verbosity >= VERB_ALGO) {
7643                         verbose(VERB_ALGO, "zonemd scheme=%d hashalgo=%d",
7644                                 scheme, hashalgo);
7645                         log_hex("ZONEMD should be  ", gen, genlen);
7646                         log_hex("ZONEMD to check is", hash, hashlen);
7647                 }
7648                 return 0;
7649         }
7650         /* check digest */
7651         if(memcmp(hash, gen, genlen) != 0) {
7652                 *reason = "incorrect digest";
7653                 if(verbosity >= VERB_ALGO) {
7654                         verbose(VERB_ALGO, "zonemd scheme=%d hashalgo=%d",
7655                                 scheme, hashalgo);
7656                         log_hex("ZONEMD should be  ", gen, genlen);
7657                         log_hex("ZONEMD to check is", hash, hashlen);
7658                 }
7659                 return 0;
7660         }
7661         return 1;
7662 }
7663
7664 /** log auth zone message with zone name in front. */
7665 static void auth_zone_log(uint8_t* name, enum verbosity_value level,
7666         const char* format, ...) ATTR_FORMAT(printf, 3, 4);
7667 static void auth_zone_log(uint8_t* name, enum verbosity_value level,
7668         const char* format, ...)
7669 {
7670         va_list args;
7671         va_start(args, format);
7672         if(verbosity >= level) {
7673                 char str[255+1];
7674                 char msg[MAXSYSLOGMSGLEN];
7675                 dname_str(name, str);
7676                 vsnprintf(msg, sizeof(msg), format, args);
7677                 verbose(level, "auth zone %s %s", str, msg);
7678         }
7679         va_end(args);
7680 }
7681
7682 /** ZONEMD, dnssec verify the rrset with the dnskey */
7683 static int zonemd_dnssec_verify_rrset(struct auth_zone* z,
7684         struct module_env* env, struct module_stack* mods,
7685         struct ub_packed_rrset_key* dnskey, struct auth_data* node,
7686         struct auth_rrset* rrset, char** why_bogus)
7687 {
7688         struct ub_packed_rrset_key pk;
7689         enum sec_status sec;
7690         struct val_env* ve;
7691         int m;
7692         m = modstack_find(mods, "validator");
7693         if(m == -1) {
7694                 auth_zone_log(z->name, VERB_ALGO, "zonemd dnssec verify: have "
7695                         "DNSKEY chain of trust, but no validator module");
7696                 return 0;
7697         }
7698         ve = (struct val_env*)env->modinfo[m];
7699
7700         memset(&pk, 0, sizeof(pk));
7701         pk.entry.key = &pk;
7702         pk.entry.data = rrset->data;
7703         pk.rk.dname = node->name;
7704         pk.rk.dname_len = node->namelen;
7705         pk.rk.type = htons(rrset->type);
7706         pk.rk.rrset_class = htons(z->dclass);
7707         if(verbosity >= VERB_ALGO) {
7708                 char typestr[32];
7709                 typestr[0]=0;
7710                 sldns_wire2str_type_buf(rrset->type, typestr, sizeof(typestr));
7711                 auth_zone_log(z->name, VERB_ALGO,
7712                         "zonemd: verify %s RRset with DNSKEY", typestr);
7713         }
7714         sec = dnskeyset_verify_rrset(env, ve, &pk, dnskey, NULL, why_bogus,
7715                 LDNS_SECTION_ANSWER, NULL);
7716         if(sec == sec_status_secure) {
7717                 return 1;
7718         }
7719         if(why_bogus)
7720                 auth_zone_log(z->name, VERB_ALGO, "DNSSEC verify was bogus: %s", *why_bogus);
7721         return 0;
7722 }
7723
7724 /** check for nsec3, the RR with params equal, if bitmap has the type */
7725 static int nsec3_of_param_has_type(struct auth_rrset* nsec3, int algo,
7726         size_t iter, uint8_t* salt, size_t saltlen, uint16_t rrtype)
7727 {
7728         int i, count = (int)nsec3->data->count;
7729         struct ub_packed_rrset_key pk;
7730         memset(&pk, 0, sizeof(pk));
7731         pk.entry.data = nsec3->data;
7732         for(i=0; i<count; i++) {
7733                 int rralgo;
7734                 size_t rriter, rrsaltlen;
7735                 uint8_t* rrsalt;
7736                 if(!nsec3_get_params(&pk, i, &rralgo, &rriter, &rrsalt,
7737                         &rrsaltlen))
7738                         continue; /* no parameters, malformed */
7739                 if(rralgo != algo || rriter != iter || rrsaltlen != saltlen)
7740                         continue; /* different parameters */
7741                 if(saltlen != 0) {
7742                         if(rrsalt == NULL || salt == NULL)
7743                                 continue;
7744                         if(memcmp(rrsalt, salt, saltlen) != 0)
7745                                 continue; /* different salt parameters */
7746                 }
7747                 if(nsec3_has_type(&pk, i, rrtype))
7748                         return 1;
7749         }
7750         return 0;
7751 }
7752
7753 /** Verify the absence of ZONEMD with DNSSEC by checking NSEC, NSEC3 type flag.
7754  * return false on failure, reason contains description of failure. */
7755 static int zonemd_check_dnssec_absence(struct auth_zone* z,
7756         struct module_env* env, struct module_stack* mods,
7757         struct ub_packed_rrset_key* dnskey, struct auth_data* apex,
7758         char** reason, char** why_bogus)
7759 {
7760         struct auth_rrset* nsec = NULL;
7761         if(!apex) {
7762                 *reason = "zone has no apex domain but ZONEMD missing";
7763                 return 0;
7764         }
7765         nsec = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC);
7766         if(nsec) {
7767                 struct ub_packed_rrset_key pk;
7768                 /* dnssec verify the NSEC */
7769                 if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex,
7770                         nsec, why_bogus)) {
7771                         *reason = "DNSSEC verify failed for NSEC RRset";
7772                         return 0;
7773                 }
7774                 /* check type bitmap */
7775                 memset(&pk, 0, sizeof(pk));
7776                 pk.entry.data = nsec->data;
7777                 if(nsec_has_type(&pk, LDNS_RR_TYPE_ZONEMD)) {
7778                         *reason = "DNSSEC NSEC bitmap says type ZONEMD exists";
7779                         return 0;
7780                 }
7781                 auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC NSEC verification of absence of ZONEMD secure");
7782         } else {
7783                 /* NSEC3 perhaps ? */
7784                 int algo;
7785                 size_t iter, saltlen;
7786                 uint8_t* salt;
7787                 struct auth_rrset* nsec3param = az_domain_rrset(apex,
7788                         LDNS_RR_TYPE_NSEC3PARAM);
7789                 struct auth_data* match;
7790                 struct auth_rrset* nsec3;
7791                 if(!nsec3param) {
7792                         *reason = "zone has no NSEC information but ZONEMD missing";
7793                         return 0;
7794                 }
7795                 if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen)) {
7796                         *reason = "zone has no NSEC information but ZONEMD missing";
7797                         return 0;
7798                 }
7799                 /* find the NSEC3 record */
7800                 match = az_nsec3_find_exact(z, z->name, z->namelen, algo,
7801                         iter, salt, saltlen);
7802                 if(!match) {
7803                         *reason = "zone has no NSEC3 domain for the apex but ZONEMD missing";
7804                         return 0;
7805                 }
7806                 nsec3 = az_domain_rrset(match, LDNS_RR_TYPE_NSEC3);
7807                 if(!nsec3) {
7808                         *reason = "zone has no NSEC3 RRset for the apex but ZONEMD missing";
7809                         return 0;
7810                 }
7811                 /* dnssec verify the NSEC3 */
7812                 if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, match,
7813                         nsec3, why_bogus)) {
7814                         *reason = "DNSSEC verify failed for NSEC3 RRset";
7815                         return 0;
7816                 }
7817                 /* check type bitmap */
7818                 if(nsec3_of_param_has_type(nsec3, algo, iter, salt, saltlen,
7819                         LDNS_RR_TYPE_ZONEMD)) {
7820                         *reason = "DNSSEC NSEC3 bitmap says type ZONEMD exists";
7821                         return 0;
7822                 }
7823                 auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC NSEC3 verification of absence of ZONEMD secure");
7824         }
7825
7826         return 1;
7827 }
7828
7829 /** Verify the SOA and ZONEMD DNSSEC signatures.
7830  * return false on failure, reason contains description of failure. */
7831 static int zonemd_check_dnssec_soazonemd(struct auth_zone* z,
7832         struct module_env* env, struct module_stack* mods,
7833         struct ub_packed_rrset_key* dnskey, struct auth_data* apex,
7834         struct auth_rrset* zonemd_rrset, char** reason, char** why_bogus)
7835 {
7836         struct auth_rrset* soa;
7837         if(!apex) {
7838                 *reason = "zone has no apex domain";
7839                 return 0;
7840         }
7841         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
7842         if(!soa) {
7843                 *reason = "zone has no SOA RRset";
7844                 return 0;
7845         }
7846         if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex, soa,
7847                 why_bogus)) {
7848                 *reason = "DNSSEC verify failed for SOA RRset";
7849                 return 0;
7850         }
7851         if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex,
7852                 zonemd_rrset, why_bogus)) {
7853                 *reason = "DNSSEC verify failed for ZONEMD RRset";
7854                 return 0;
7855         }
7856         auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC verification of SOA and ZONEMD RRsets secure");
7857         return 1;
7858 }
7859
7860 /**
7861  * Fail the ZONEMD verification.
7862  * @param z: auth zone that fails.
7863  * @param env: environment with config, to ignore failure or not.
7864  * @param reason: failure string description.
7865  * @param why_bogus: failure string for DNSSEC verification failure.
7866  * @param result: strdup result in here if not NULL.
7867  */
7868 static void auth_zone_zonemd_fail(struct auth_zone* z, struct module_env* env,
7869         char* reason, char* why_bogus, char** result)
7870 {
7871         char zstr[255+1];
7872         /* if fail: log reason, and depending on config also take action
7873          * and drop the zone, eg. it is gone from memory, set zone_expired */
7874         dname_str(z->name, zstr);
7875         if(!reason) reason = "verification failed";
7876         if(result) {
7877                 if(why_bogus) {
7878                         char res[1024];
7879                         snprintf(res, sizeof(res), "%s: %s", reason,
7880                                 why_bogus);
7881                         *result = strdup(res);
7882                 } else {
7883                         *result = strdup(reason);
7884                 }
7885                 if(!*result) log_err("out of memory");
7886         } else {
7887                 log_warn("auth zone %s: ZONEMD verification failed: %s", zstr, reason);
7888         }
7889
7890         if(env->cfg->zonemd_permissive_mode) {
7891                 verbose(VERB_ALGO, "zonemd-permissive-mode enabled, "
7892                         "not blocking zone %s", zstr);
7893                 return;
7894         }
7895
7896         /* expired means the zone gives servfail and is not used by
7897          * lookup if fallback_enabled*/
7898         z->zone_expired = 1;
7899 }
7900
7901 /**
7902  * Verify the zonemd with DNSSEC and hash check, with given key.
7903  * @param z: auth zone.
7904  * @param env: environment with config and temp buffers.
7905  * @param mods: module stack with validator env for verification.
7906  * @param dnskey: dnskey that we can use, or NULL.  If nonnull, the key
7907  *      has been verified and is the start of the chain of trust.
7908  * @param is_insecure: if true, the dnskey is not used, the zone is insecure.
7909  *      And dnssec is not used.  It is DNSSEC secure insecure or not under
7910  *      a trust anchor.
7911  * @param result: if not NULL result reason copied here.
7912  */
7913 static void
7914 auth_zone_verify_zonemd_with_key(struct auth_zone* z, struct module_env* env,
7915         struct module_stack* mods, struct ub_packed_rrset_key* dnskey,
7916         int is_insecure, char** result)
7917 {
7918         char* reason = NULL, *why_bogus = NULL;
7919         struct auth_data* apex = NULL;
7920         struct auth_rrset* zonemd_rrset = NULL;
7921         int zonemd_absent = 0, zonemd_absence_dnssecok = 0;
7922
7923         /* see if ZONEMD is present or absent. */
7924         apex = az_find_name(z, z->name, z->namelen);
7925         if(!apex) {
7926                 zonemd_absent = 1;
7927         } else {
7928                 zonemd_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_ZONEMD);
7929                 if(!zonemd_rrset || zonemd_rrset->data->count==0) {
7930                         zonemd_absent = 1;
7931                         zonemd_rrset = NULL;
7932                 }
7933         }
7934
7935         /* if no DNSSEC, done. */
7936         /* if no ZONEMD, and DNSSEC, use DNSKEY to verify NSEC or NSEC3 for
7937          * zone apex.  Check ZONEMD bit is turned off or else fail */
7938         /* if ZONEMD, and DNSSEC, check DNSSEC signature on SOA and ZONEMD,
7939          * or else fail */
7940         if(!dnskey && !is_insecure) {
7941                 auth_zone_zonemd_fail(z, env, "DNSKEY missing", NULL, result);
7942                 return;
7943         } else if(!zonemd_rrset && dnskey && !is_insecure) {
7944                 /* fetch, DNSSEC verify, and check NSEC/NSEC3 */
7945                 if(!zonemd_check_dnssec_absence(z, env, mods, dnskey, apex,
7946                         &reason, &why_bogus)) {
7947                         auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
7948                         return;
7949                 }
7950                 zonemd_absence_dnssecok = 1;
7951         } else if(zonemd_rrset && dnskey && !is_insecure) {
7952                 /* check DNSSEC verify of SOA and ZONEMD */
7953                 if(!zonemd_check_dnssec_soazonemd(z, env, mods, dnskey, apex,
7954                         zonemd_rrset, &reason, &why_bogus)) {
7955                         auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
7956                         return;
7957                 }
7958         }
7959
7960         if(zonemd_absent && z->zonemd_reject_absence) {
7961                 auth_zone_zonemd_fail(z, env, "ZONEMD absent and that is not allowed by config", NULL, result);
7962                 return;
7963         }
7964         if(zonemd_absent && zonemd_absence_dnssecok) {
7965                 auth_zone_log(z->name, VERB_ALGO, "DNSSEC verified nonexistence of ZONEMD");
7966                 if(result) {
7967                         *result = strdup("DNSSEC verified nonexistence of ZONEMD");
7968                         if(!*result) log_err("out of memory");
7969                 }
7970                 return;
7971         }
7972         if(zonemd_absent) {
7973                 auth_zone_log(z->name, VERB_ALGO, "no ZONEMD present");
7974                 if(result) {
7975                         *result = strdup("no ZONEMD present");
7976                         if(!*result) log_err("out of memory");
7977                 }
7978                 return;
7979         }
7980
7981         /* check ZONEMD checksum and report or else fail. */
7982         if(!auth_zone_zonemd_check_hash(z, env, &reason)) {
7983                 auth_zone_zonemd_fail(z, env, reason, NULL, result);
7984                 return;
7985         }
7986
7987         /* success! log the success */
7988         auth_zone_log(z->name, VERB_ALGO, "ZONEMD verification successful");
7989         if(result) {
7990                 *result = strdup("ZONEMD verification successful");
7991                 if(!*result) log_err("out of memory");
7992         }
7993 }
7994
7995 /**
7996  * verify the zone DNSKEY rrset from the trust anchor
7997  * This is possible because the anchor is for the zone itself, and can
7998  * thus apply straight to the zone DNSKEY set.
7999  * @param z: the auth zone.
8000  * @param env: environment with time and temp buffers.
8001  * @param mods: module stack for validator environment for dnssec validation.
8002  * @param anchor: trust anchor to use
8003  * @param is_insecure: returned, true if the zone is securely insecure.
8004  * @param why_bogus: if the routine fails, returns the failure reason.
8005  * @param keystorage: where to store the ub_packed_rrset_key that is created
8006  *      on success. A pointer to it is returned on success.
8007  * @return the dnskey RRset, reference to zone data and keystorage, or
8008  *      NULL on failure.
8009  */
8010 static struct ub_packed_rrset_key*
8011 zonemd_get_dnskey_from_anchor(struct auth_zone* z, struct module_env* env,
8012         struct module_stack* mods, struct trust_anchor* anchor,
8013         int* is_insecure, char** why_bogus,
8014         struct ub_packed_rrset_key* keystorage)
8015 {
8016         struct auth_data* apex;
8017         struct auth_rrset* dnskey_rrset;
8018         enum sec_status sec;
8019         struct val_env* ve;
8020         int m;
8021
8022         apex = az_find_name(z, z->name, z->namelen);
8023         if(!apex) {
8024                 *why_bogus = "have trust anchor, but zone has no apex domain for DNSKEY";
8025                 return 0;
8026         }
8027         dnskey_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_DNSKEY);
8028         if(!dnskey_rrset || dnskey_rrset->data->count==0) {
8029                 *why_bogus = "have trust anchor, but zone has no DNSKEY";
8030                 return 0;
8031         }
8032
8033         m = modstack_find(mods, "validator");
8034         if(m == -1) {
8035                 *why_bogus = "have trust anchor, but no validator module";
8036                 return 0;
8037         }
8038         ve = (struct val_env*)env->modinfo[m];
8039
8040         memset(keystorage, 0, sizeof(*keystorage));
8041         keystorage->entry.key = keystorage;
8042         keystorage->entry.data = dnskey_rrset->data;
8043         keystorage->rk.dname = apex->name;
8044         keystorage->rk.dname_len = apex->namelen;
8045         keystorage->rk.type = htons(LDNS_RR_TYPE_DNSKEY);
8046         keystorage->rk.rrset_class = htons(z->dclass);
8047         auth_zone_log(z->name, VERB_QUERY,
8048                 "zonemd: verify DNSKEY RRset with trust anchor");
8049         sec = val_verify_DNSKEY_with_TA(env, ve, keystorage, anchor->ds_rrset,
8050                 anchor->dnskey_rrset, NULL, why_bogus, NULL);
8051         regional_free_all(env->scratch);
8052         if(sec == sec_status_secure) {
8053                 /* success */
8054                 *is_insecure = 0;
8055                 return keystorage;
8056         } else if(sec == sec_status_insecure) {
8057                 /* insecure */
8058                 *is_insecure = 1;
8059         } else {
8060                 /* bogus */
8061                 *is_insecure = 0;
8062                 auth_zone_log(z->name, VERB_ALGO,
8063                         "zonemd: verify DNSKEY RRset with trust anchor failed: %s", *why_bogus);
8064         }
8065         return NULL;
8066 }
8067
8068 /** callback for ZONEMD lookup of DNSKEY */
8069 void auth_zonemd_dnskey_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
8070         enum sec_status sec, char* why_bogus, int ATTR_UNUSED(was_ratelimited))
8071 {
8072         struct auth_zone* z = (struct auth_zone*)arg;
8073         struct module_env* env;
8074         char* reason = NULL;
8075         struct ub_packed_rrset_key* dnskey = NULL;
8076         int is_insecure = 0;
8077
8078         lock_rw_wrlock(&z->lock);
8079         env = z->zonemd_callback_env;
8080         /* release the env variable so another worker can pick up the
8081          * ZONEMD verification task if it wants to */
8082         z->zonemd_callback_env = NULL;
8083         if(!env || env->outnet->want_to_quit || z->zone_deleted) {
8084                 lock_rw_unlock(&z->lock);
8085                 return; /* stop on quit */
8086         }
8087
8088         /* process result */
8089         if(sec == sec_status_bogus) {
8090                 reason = why_bogus;
8091                 if(!reason)
8092                         reason = "lookup of DNSKEY was bogus";
8093                 auth_zone_log(z->name, VERB_ALGO,
8094                         "zonemd lookup of DNSKEY was bogus: %s", reason);
8095         } else if(rcode == LDNS_RCODE_NOERROR) {
8096                 uint16_t wanted_qtype = LDNS_RR_TYPE_DNSKEY;
8097                 struct regional* temp = env->scratch;
8098                 struct query_info rq;
8099                 struct reply_info* rep;
8100                 memset(&rq, 0, sizeof(rq));
8101                 rep = parse_reply_in_temp_region(buf, temp, &rq);
8102                 if(rep && rq.qtype == wanted_qtype &&
8103                         query_dname_compare(z->name, rq.qname) == 0 &&
8104                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
8105                         /* parsed successfully */
8106                         struct ub_packed_rrset_key* answer =
8107                                 reply_find_answer_rrset(&rq, rep);
8108                         if(answer && sec == sec_status_secure) {
8109                                 dnskey = answer;
8110                                 auth_zone_log(z->name, VERB_ALGO,
8111                                         "zonemd lookup of DNSKEY was secure");
8112                         } else if(sec == sec_status_secure && !answer) {
8113                                 is_insecure = 1;
8114                                 auth_zone_log(z->name, VERB_ALGO,
8115                                         "zonemd lookup of DNSKEY has no content, but is secure, treat as insecure");
8116                         } else if(sec == sec_status_insecure) {
8117                                 is_insecure = 1;
8118                                 auth_zone_log(z->name, VERB_ALGO,
8119                                         "zonemd lookup of DNSKEY was insecure");
8120                         } else if(sec == sec_status_indeterminate) {
8121                                 is_insecure = 1;
8122                                 auth_zone_log(z->name, VERB_ALGO,
8123                                         "zonemd lookup of DNSKEY was indeterminate, treat as insecure");
8124                         } else {
8125                                 auth_zone_log(z->name, VERB_ALGO,
8126                                         "zonemd lookup of DNSKEY has nodata");
8127                                 reason = "lookup of DNSKEY has nodata";
8128                         }
8129                 } else if(rep && rq.qtype == wanted_qtype &&
8130                         query_dname_compare(z->name, rq.qname) == 0 &&
8131                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8132                         sec == sec_status_secure) {
8133                         /* secure nxdomain, so the zone is like some RPZ zone
8134                          * that does not exist in the wider internet, with
8135                          * a secure nxdomain answer outside of it. So we
8136                          * treat the zonemd zone without a dnssec chain of
8137                          * trust, as insecure. */
8138                         is_insecure = 1;
8139                         auth_zone_log(z->name, VERB_ALGO,
8140                                 "zonemd lookup of DNSKEY was secure NXDOMAIN, treat as insecure");
8141                 } else if(rep && rq.qtype == wanted_qtype &&
8142                         query_dname_compare(z->name, rq.qname) == 0 &&
8143                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8144                         sec == sec_status_insecure) {
8145                         is_insecure = 1;
8146                         auth_zone_log(z->name, VERB_ALGO,
8147                                 "zonemd lookup of DNSKEY was insecure NXDOMAIN, treat as insecure");
8148                 } else if(rep && rq.qtype == wanted_qtype &&
8149                         query_dname_compare(z->name, rq.qname) == 0 &&
8150                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8151                         sec == sec_status_indeterminate) {
8152                         is_insecure = 1;
8153                         auth_zone_log(z->name, VERB_ALGO,
8154                                 "zonemd lookup of DNSKEY was indeterminate NXDOMAIN, treat as insecure");
8155                 } else {
8156                         auth_zone_log(z->name, VERB_ALGO,
8157                                 "zonemd lookup of DNSKEY has no answer");
8158                         reason = "lookup of DNSKEY has no answer";
8159                 }
8160         } else {
8161                 auth_zone_log(z->name, VERB_ALGO,
8162                         "zonemd lookup of DNSKEY failed");
8163                 reason = "lookup of DNSKEY failed";
8164         }
8165
8166         if(reason) {
8167                 auth_zone_zonemd_fail(z, env, reason, NULL, NULL);
8168                 lock_rw_unlock(&z->lock);
8169                 return;
8170         }
8171
8172         auth_zone_verify_zonemd_with_key(z, env, &env->mesh->mods, dnskey,
8173                 is_insecure, NULL);
8174         regional_free_all(env->scratch);
8175         lock_rw_unlock(&z->lock);
8176 }
8177
8178 /** lookup DNSKEY for ZONEMD verification */
8179 static int
8180 zonemd_lookup_dnskey(struct auth_zone* z, struct module_env* env)
8181 {
8182         struct query_info qinfo;
8183         uint16_t qflags = BIT_RD;
8184         struct edns_data edns;
8185         sldns_buffer* buf = env->scratch_buffer;
8186
8187         if(z->zonemd_callback_env) {
8188                 /* another worker is already working on the callback
8189                  * for the DNSKEY lookup for ZONEMD verification.
8190                  * We do not also have to do ZONEMD verification, let that
8191                  * worker do it */
8192                 auth_zone_log(z->name, VERB_ALGO,
8193                         "zonemd needs lookup of DNSKEY and that already worked on by another worker");
8194                 return 1;
8195         }
8196
8197         /* use mesh_new_callback to lookup the DNSKEY,
8198          * and then wait for them to be looked up (in cache, or query) */
8199         qinfo.qname_len = z->namelen;
8200         qinfo.qname = z->name;
8201         qinfo.qclass = z->dclass;
8202         qinfo.qtype = LDNS_RR_TYPE_DNSKEY;
8203         qinfo.local_alias = NULL;
8204         if(verbosity >= VERB_ALGO) {
8205                 char buf1[512];
8206                 char buf2[LDNS_MAX_DOMAINLEN+1];
8207                 dname_str(z->name, buf2);
8208                 snprintf(buf1, sizeof(buf1), "auth zone %s: lookup DNSKEY "
8209                         "for zonemd verification", buf2);
8210                 log_query_info(VERB_ALGO, buf1, &qinfo);
8211         }
8212         edns.edns_present = 1;
8213         edns.ext_rcode = 0;
8214         edns.edns_version = 0;
8215         edns.bits = EDNS_DO;
8216         edns.opt_list = NULL;
8217         if(sldns_buffer_capacity(buf) < 65535)
8218                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
8219         else    edns.udp_size = 65535;
8220
8221         /* store the worker-specific module env for the callback.
8222          * We can then reference this when the callback executes */
8223         z->zonemd_callback_env = env;
8224         /* the callback can be called straight away */
8225         lock_rw_unlock(&z->lock);
8226         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
8227                 &auth_zonemd_dnskey_lookup_callback, z)) {
8228                 lock_rw_wrlock(&z->lock);
8229                 log_err("out of memory lookup up dnskey for zonemd");
8230                 return 0;
8231         }
8232         lock_rw_wrlock(&z->lock);
8233         return 1;
8234 }
8235
8236 void auth_zone_verify_zonemd(struct auth_zone* z, struct module_env* env,
8237         struct module_stack* mods, char** result, int offline, int only_online)
8238 {
8239         char* reason = NULL, *why_bogus = NULL;
8240         struct trust_anchor* anchor = NULL;
8241         struct ub_packed_rrset_key* dnskey = NULL;
8242         struct ub_packed_rrset_key keystorage;
8243         int is_insecure = 0;
8244         /* verify the ZONEMD if present.
8245          * If not present check if absence is allowed by DNSSEC */
8246         if(!z->zonemd_check)
8247                 return;
8248
8249         /* if zone is under a trustanchor */
8250         /* is it equal to trustanchor - get dnskey's verified */
8251         /* else, find chain of trust by fetching DNSKEYs lookup for zone */
8252         /* result if that, if insecure, means no DNSSEC for the ZONEMD,
8253          * otherwise we have the zone DNSKEY for the DNSSEC verification. */
8254         if(env->anchors)
8255                 anchor = anchors_lookup(env->anchors, z->name, z->namelen,
8256                         z->dclass);
8257         if(anchor && anchor->numDS == 0 && anchor->numDNSKEY == 0) {
8258                 /* domain-insecure trust anchor for unsigned zones */
8259                 lock_basic_unlock(&anchor->lock);
8260                 if(only_online)
8261                         return;
8262                 dnskey = NULL;
8263                 is_insecure = 1;
8264         } else if(anchor && query_dname_compare(z->name, anchor->name) == 0) {
8265                 if(only_online) {
8266                         lock_basic_unlock(&anchor->lock);
8267                         return;
8268                 }
8269                 /* equal to trustanchor, no need for online lookups */
8270                 dnskey = zonemd_get_dnskey_from_anchor(z, env, mods, anchor,
8271                         &is_insecure, &why_bogus, &keystorage);
8272                 lock_basic_unlock(&anchor->lock);
8273                 if(!dnskey && !reason && !is_insecure) {
8274                         reason = "verify DNSKEY RRset with trust anchor failed";
8275                 }
8276         } else if(anchor) {
8277                 lock_basic_unlock(&anchor->lock);
8278                 /* perform online lookups */
8279                 if(offline)
8280                         return;
8281                 /* setup online lookups, and wait for them */
8282                 if(zonemd_lookup_dnskey(z, env)) {
8283                         /* wait for the lookup */
8284                         return;
8285                 }
8286                 reason = "could not lookup DNSKEY for chain of trust";
8287         } else {
8288                 /* the zone is not under a trust anchor */
8289                 if(only_online)
8290                         return;
8291                 dnskey = NULL;
8292                 is_insecure = 1;
8293         }
8294
8295         if(reason) {
8296                 auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
8297                 return;
8298         }
8299
8300         auth_zone_verify_zonemd_with_key(z, env, mods, dnskey, is_insecure,
8301                 result);
8302         regional_free_all(env->scratch);
8303 }
8304
8305 void auth_zones_pickup_zonemd_verify(struct auth_zones* az,
8306         struct module_env* env)
8307 {
8308         struct auth_zone key;
8309         uint8_t savezname[255+1];
8310         size_t savezname_len;
8311         struct auth_zone* z;
8312         key.node.key = &key;
8313         lock_rw_rdlock(&az->lock);
8314         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
8315                 lock_rw_wrlock(&z->lock);
8316                 if(!z->zonemd_check) {
8317                         lock_rw_unlock(&z->lock);
8318                         continue;
8319                 }
8320                 key.dclass = z->dclass;
8321                 key.namelabs = z->namelabs;
8322                 if(z->namelen > sizeof(savezname)) {
8323                         lock_rw_unlock(&z->lock);
8324                         log_err("auth_zones_pickup_zonemd_verify: zone name too long");
8325                         continue;
8326                 }
8327                 savezname_len = z->namelen;
8328                 memmove(savezname, z->name, z->namelen);
8329                 lock_rw_unlock(&az->lock);
8330                 auth_zone_verify_zonemd(z, env, &env->mesh->mods, NULL, 0, 1);
8331                 lock_rw_unlock(&z->lock);
8332                 lock_rw_rdlock(&az->lock);
8333                 /* find the zone we had before, it is not deleted,
8334                  * because we have a flag for that that is processed at
8335                  * apply_cfg time */
8336                 key.namelen = savezname_len;
8337                 key.name = savezname;
8338                 z = (struct auth_zone*)rbtree_search(&az->ztree, &key);
8339                 if(!z)
8340                         break;
8341         }
8342         lock_rw_unlock(&az->lock);
8343 }