contrib/unbound/services/authzone.c

   1 /*
   2  * services/authzone.c - authoritative zone that is locally hosted.
   3  *
   4  * Copyright (c) 2017, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains the functions for an authority zone.  This zone
  40  * is queried by the iterator, just like a stub or forward zone, but then
  41  * the data is locally held.
  42  */
  43
  44 #include "config.h"
  45 #include "services/authzone.h"
  46 #include "util/data/dname.h"
  47 #include "util/data/msgparse.h"
  48 #include "util/data/msgreply.h"
  49 #include "util/data/msgencode.h"
  50 #include "util/data/packed_rrset.h"
  51 #include "util/regional.h"
  52 #include "util/net_help.h"
  53 #include "util/netevent.h"
  54 #include "util/config_file.h"
  55 #include "util/log.h"
  56 #include "util/module.h"
  57 #include "util/random.h"
  58 #include "services/cache/dns.h"
  59 #include "services/outside_network.h"
  60 #include "services/listen_dnsport.h"
  61 #include "services/mesh.h"
  62 #include "sldns/rrdef.h"
  63 #include "sldns/pkthdr.h"
  64 #include "sldns/sbuffer.h"
  65 #include "sldns/str2wire.h"
  66 #include "sldns/wire2str.h"
  67 #include "sldns/parseutil.h"
  68 #include "sldns/keyraw.h"
  69 #include "validator/val_nsec3.h"
  70 #include "validator/val_nsec.h"
  71 #include "validator/val_secalgo.h"
  72 #include "validator/val_sigcrypt.h"
  73 #include "validator/val_anchor.h"
  74 #include "validator/val_utils.h"
  75 #include <ctype.h>
  76
  77 /** bytes to use for NSEC3 hash buffer. 20 for sha1 */
  78 #define N3HASHBUFLEN 32
  79 /** max number of CNAMEs we are willing to follow (in one answer) */
  80 #define MAX_CNAME_CHAIN 8
  81 /** timeout for probe packets for SOA */
  82 #define AUTH_PROBE_TIMEOUT 100 /* msec */
  83 /** when to stop with SOA probes (when exponential timeouts exceed this) */
  84 #define AUTH_PROBE_TIMEOUT_STOP 1000 /* msec */
  85 /* auth transfer timeout for TCP connections, in msec */
  86 #define AUTH_TRANSFER_TIMEOUT 10000 /* msec */
  87 /* auth transfer max backoff for failed transfers and probes */
  88 #define AUTH_TRANSFER_MAX_BACKOFF 86400 /* sec */
  89 /* auth http port number */
  90 #define AUTH_HTTP_PORT 80
  91 /* auth https port number */
  92 #define AUTH_HTTPS_PORT 443
  93 /* max depth for nested $INCLUDEs */
  94 #define MAX_INCLUDE_DEPTH 10
  95 /** number of timeouts before we fallback from IXFR to AXFR,
  96  * because some versions of servers (eg. dnsmasq) drop IXFR packets. */
  97 #define NUM_TIMEOUTS_FALLBACK_IXFR 3
  98
  99 /** pick up nextprobe task to start waiting to perform transfer actions */
 100 static void xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
 101         int failure, int lookup_only);
 102 /** move to sending the probe packets, next if fails. task_probe */
 103 static void xfr_probe_send_or_end(struct auth_xfer* xfr,
 104         struct module_env* env);
 105 /** pick up probe task with specified(or NULL) destination first,
 106  * or transfer task if nothing to probe, or false if already in progress */
 107 static int xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
 108         struct auth_master* spec);
 109 /** delete xfer structure (not its tree entry) */
 110 static void auth_xfer_delete(struct auth_xfer* xfr);
 111
 112 /** create new dns_msg */
 113 static struct dns_msg*
 114 msg_create(struct regional* region, struct query_info* qinfo)
 115 {
 116         struct dns_msg* msg = (struct dns_msg*)regional_alloc(region,
 117                 sizeof(struct dns_msg));
 118         if(!msg)
 119                 return NULL;
 120         msg->qinfo.qname = regional_alloc_init(region, qinfo->qname,
 121                 qinfo->qname_len);
 122         if(!msg->qinfo.qname)
 123                 return NULL;
 124         msg->qinfo.qname_len = qinfo->qname_len;
 125         msg->qinfo.qtype = qinfo->qtype;
 126         msg->qinfo.qclass = qinfo->qclass;
 127         msg->qinfo.local_alias = NULL;
 128         /* non-packed reply_info, because it needs to grow the array */
 129         msg->rep = (struct reply_info*)regional_alloc_zero(region,
 130                 sizeof(struct reply_info)-sizeof(struct rrset_ref));
 131         if(!msg->rep)
 132                 return NULL;
 133         msg->rep->flags = (uint16_t)(BIT_QR | BIT_AA);
 134         msg->rep->authoritative = 1;
 135         msg->rep->qdcount = 1;
 136         /* rrsets is NULL, no rrsets yet */
 137         return msg;
 138 }
 139
 140 /** grow rrset array by one in msg */
 141 static int
 142 msg_grow_array(struct regional* region, struct dns_msg* msg)
 143 {
 144         if(msg->rep->rrsets == NULL) {
 145                 msg->rep->rrsets = regional_alloc_zero(region,
 146                         sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
 147                 if(!msg->rep->rrsets)
 148                         return 0;
 149         } else {
 150                 struct ub_packed_rrset_key** rrsets_old = msg->rep->rrsets;
 151                 msg->rep->rrsets = regional_alloc_zero(region,
 152                         sizeof(struct ub_packed_rrset_key*)*(msg->rep->rrset_count+1));
 153                 if(!msg->rep->rrsets)
 154                         return 0;
 155                 memmove(msg->rep->rrsets, rrsets_old,
 156                         sizeof(struct ub_packed_rrset_key*)*msg->rep->rrset_count);
 157         }
 158         return 1;
 159 }
 160
 161 /** get ttl of rrset */
 162 static time_t
 163 get_rrset_ttl(struct ub_packed_rrset_key* k)
 164 {
 165         struct packed_rrset_data* d = (struct packed_rrset_data*)
 166                 k->entry.data;
 167         return d->ttl;
 168 }
 169
 170 /** Copy rrset into region from domain-datanode and packet rrset */
 171 static struct ub_packed_rrset_key*
 172 auth_packed_rrset_copy_region(struct auth_zone* z, struct auth_data* node,
 173         struct auth_rrset* rrset, struct regional* region, time_t adjust)
 174 {
 175         struct ub_packed_rrset_key key;
 176         memset(&key, 0, sizeof(key));
 177         key.entry.key = &key;
 178         key.entry.data = rrset->data;
 179         key.rk.dname = node->name;
 180         key.rk.dname_len = node->namelen;
 181         key.rk.type = htons(rrset->type);
 182         key.rk.rrset_class = htons(z->dclass);
 183         key.entry.hash = rrset_key_hash(&key.rk);
 184         return packed_rrset_copy_region(&key, region, adjust);
 185 }
 186
 187 /** fix up msg->rep TTL and prefetch ttl */
 188 static void
 189 msg_ttl(struct dns_msg* msg)
 190 {
 191         if(msg->rep->rrset_count == 0) return;
 192         if(msg->rep->rrset_count == 1) {
 193                 msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
 194                 msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 195                 msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
 196         } else if(get_rrset_ttl(msg->rep->rrsets[msg->rep->rrset_count-1]) <
 197                 msg->rep->ttl) {
 198                 msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[
 199                         msg->rep->rrset_count-1]);
 200                 msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 201                 msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
 202         }
 203 }
 204
 205 /** see if rrset is a duplicate in the answer message */
 206 static int
 207 msg_rrset_duplicate(struct dns_msg* msg, uint8_t* nm, size_t nmlen,
 208         uint16_t type, uint16_t dclass)
 209 {
 210         size_t i;
 211         for(i=0; i<msg->rep->rrset_count; i++) {
 212                 struct ub_packed_rrset_key* k = msg->rep->rrsets[i];
 213                 if(ntohs(k->rk.type) == type && k->rk.dname_len == nmlen &&
 214                         ntohs(k->rk.rrset_class) == dclass &&
 215                         query_dname_compare(k->rk.dname, nm) == 0)
 216                         return 1;
 217         }
 218         return 0;
 219 }
 220
 221 /** add rrset to answer section (no auth, add rrsets yet) */
 222 static int
 223 msg_add_rrset_an(struct auth_zone* z, struct regional* region,
 224         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 225 {
 226         log_assert(msg->rep->ns_numrrsets == 0);
 227         log_assert(msg->rep->ar_numrrsets == 0);
 228         if(!rrset || !node)
 229                 return 1;
 230         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 231                 z->dclass))
 232                 return 1;
 233         /* grow array */
 234         if(!msg_grow_array(region, msg))
 235                 return 0;
 236         /* copy it */
 237         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 238                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 239                 return 0;
 240         msg->rep->rrset_count++;
 241         msg->rep->an_numrrsets++;
 242         msg_ttl(msg);
 243         return 1;
 244 }
 245
 246 /** add rrset to authority section (no additional section rrsets yet) */
 247 static int
 248 msg_add_rrset_ns(struct auth_zone* z, struct regional* region,
 249         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 250 {
 251         log_assert(msg->rep->ar_numrrsets == 0);
 252         if(!rrset || !node)
 253                 return 1;
 254         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 255                 z->dclass))
 256                 return 1;
 257         /* grow array */
 258         if(!msg_grow_array(region, msg))
 259                 return 0;
 260         /* copy it */
 261         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 262                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 263                 return 0;
 264         msg->rep->rrset_count++;
 265         msg->rep->ns_numrrsets++;
 266         msg_ttl(msg);
 267         return 1;
 268 }
 269
 270 /** add rrset to additional section */
 271 static int
 272 msg_add_rrset_ar(struct auth_zone* z, struct regional* region,
 273         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 274 {
 275         if(!rrset || !node)
 276                 return 1;
 277         if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 278                 z->dclass))
 279                 return 1;
 280         /* grow array */
 281         if(!msg_grow_array(region, msg))
 282                 return 0;
 283         /* copy it */
 284         if(!(msg->rep->rrsets[msg->rep->rrset_count] =
 285                 auth_packed_rrset_copy_region(z, node, rrset, region, 0)))
 286                 return 0;
 287         msg->rep->rrset_count++;
 288         msg->rep->ar_numrrsets++;
 289         msg_ttl(msg);
 290         return 1;
 291 }
 292
 293 struct auth_zones* auth_zones_create(void)
 294 {
 295         struct auth_zones* az = (struct auth_zones*)calloc(1, sizeof(*az));
 296         if(!az) {
 297                 log_err("out of memory");
 298                 return NULL;
 299         }
 300         rbtree_init(&az->ztree, &auth_zone_cmp);
 301         rbtree_init(&az->xtree, &auth_xfer_cmp);
 302         lock_rw_init(&az->lock);
 303         lock_protect(&az->lock, &az->ztree, sizeof(az->ztree));
 304         lock_protect(&az->lock, &az->xtree, sizeof(az->xtree));
 305         /* also lock protects the rbnode's in struct auth_zone, auth_xfer */
 306         lock_rw_init(&az->rpz_lock);
 307         lock_protect(&az->rpz_lock, &az->rpz_first, sizeof(az->rpz_first));
 308         return az;
 309 }
 310
 311 int auth_zone_cmp(const void* z1, const void* z2)
 312 {
 313         /* first sort on class, so that hierarchy can be maintained within
 314          * a class */
 315         struct auth_zone* a = (struct auth_zone*)z1;
 316         struct auth_zone* b = (struct auth_zone*)z2;
 317         int m;
 318         if(a->dclass != b->dclass) {
 319                 if(a->dclass < b->dclass)
 320                         return -1;
 321                 return 1;
 322         }
 323         /* sorted such that higher zones sort before lower zones (their
 324          * contents) */
 325         return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
 326 }
 327
 328 int auth_data_cmp(const void* z1, const void* z2)
 329 {
 330         struct auth_data* a = (struct auth_data*)z1;
 331         struct auth_data* b = (struct auth_data*)z2;
 332         int m;
 333         /* canonical sort, because DNSSEC needs that */
 334         return dname_canon_lab_cmp(a->name, a->namelabs, b->name,
 335                 b->namelabs, &m);
 336 }
 337
 338 int auth_xfer_cmp(const void* z1, const void* z2)
 339 {
 340         /* first sort on class, so that hierarchy can be maintained within
 341          * a class */
 342         struct auth_xfer* a = (struct auth_xfer*)z1;
 343         struct auth_xfer* b = (struct auth_xfer*)z2;
 344         int m;
 345         if(a->dclass != b->dclass) {
 346                 if(a->dclass < b->dclass)
 347                         return -1;
 348                 return 1;
 349         }
 350         /* sorted such that higher zones sort before lower zones (their
 351          * contents) */
 352         return dname_lab_cmp(a->name, a->namelabs, b->name, b->namelabs, &m);
 353 }
 354
 355 /** delete auth rrset node */
 356 static void
 357 auth_rrset_delete(struct auth_rrset* rrset)
 358 {
 359         if(!rrset) return;
 360         free(rrset->data);
 361         free(rrset);
 362 }
 363
 364 /** delete auth data domain node */
 365 static void
 366 auth_data_delete(struct auth_data* n)
 367 {
 368         struct auth_rrset* p, *np;
 369         if(!n) return;
 370         p = n->rrsets;
 371         while(p) {
 372                 np = p->next;
 373                 auth_rrset_delete(p);
 374                 p = np;
 375         }
 376         free(n->name);
 377         free(n);
 378 }
 379
 380 /** helper traverse to delete zones */
 381 static void
 382 auth_data_del(rbnode_type* n, void* ATTR_UNUSED(arg))
 383 {
 384         struct auth_data* z = (struct auth_data*)n->key;
 385         auth_data_delete(z);
 386 }
 387
 388 /** delete an auth zone structure (tree remove must be done elsewhere) */
 389 static void
 390 auth_zone_delete(struct auth_zone* z, struct auth_zones* az)
 391 {
 392         if(!z) return;
 393         lock_rw_destroy(&z->lock);
 394         traverse_postorder(&z->data, auth_data_del, NULL);
 395
 396         if(az && z->rpz) {
 397                 /* keep RPZ linked list intact */
 398                 lock_rw_wrlock(&az->rpz_lock);
 399                 if(z->rpz_az_prev)
 400                         z->rpz_az_prev->rpz_az_next = z->rpz_az_next;
 401                 else
 402                         az->rpz_first = z->rpz_az_next;
 403                 if(z->rpz_az_next)
 404                         z->rpz_az_next->rpz_az_prev = z->rpz_az_prev;
 405                 lock_rw_unlock(&az->rpz_lock);
 406         }
 407         if(z->rpz)
 408                 rpz_delete(z->rpz);
 409         free(z->name);
 410         free(z->zonefile);
 411         free(z);
 412 }
 413
 414 struct auth_zone*
 415 auth_zone_create(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 416         uint16_t dclass)
 417 {
 418         struct auth_zone* z = (struct auth_zone*)calloc(1, sizeof(*z));
 419         if(!z) {
 420                 return NULL;
 421         }
 422         z->node.key = z;
 423         z->dclass = dclass;
 424         z->namelen = nmlen;
 425         z->namelabs = dname_count_labels(nm);
 426         z->name = memdup(nm, nmlen);
 427         if(!z->name) {
 428                 free(z);
 429                 return NULL;
 430         }
 431         rbtree_init(&z->data, &auth_data_cmp);
 432         lock_rw_init(&z->lock);
 433         lock_protect(&z->lock, &z->name, sizeof(*z)-sizeof(rbnode_type)-
 434                         sizeof(&z->rpz_az_next)-sizeof(&z->rpz_az_prev));
 435         lock_rw_wrlock(&z->lock);
 436         /* z lock protects all, except rbtree itself and the rpz linked list
 437          * pointers, which are protected using az->lock */
 438         if(!rbtree_insert(&az->ztree, &z->node)) {
 439                 lock_rw_unlock(&z->lock);
 440                 auth_zone_delete(z, NULL);
 441                 log_warn("duplicate auth zone");
 442                 return NULL;
 443         }
 444         return z;
 445 }
 446
 447 struct auth_zone*
 448 auth_zone_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 449         uint16_t dclass)
 450 {
 451         struct auth_zone key;
 452         key.node.key = &key;
 453         key.dclass = dclass;
 454         key.name = nm;
 455         key.namelen = nmlen;
 456         key.namelabs = dname_count_labels(nm);
 457         return (struct auth_zone*)rbtree_search(&az->ztree, &key);
 458 }
 459
 460 struct auth_xfer*
 461 auth_xfer_find(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 462         uint16_t dclass)
 463 {
 464         struct auth_xfer key;
 465         key.node.key = &key;
 466         key.dclass = dclass;
 467         key.name = nm;
 468         key.namelen = nmlen;
 469         key.namelabs = dname_count_labels(nm);
 470         return (struct auth_xfer*)rbtree_search(&az->xtree, &key);
 471 }
 472
 473 /** find an auth zone or sorted less-or-equal, return true if exact */
 474 static int
 475 auth_zone_find_less_equal(struct auth_zones* az, uint8_t* nm, size_t nmlen,
 476         uint16_t dclass, struct auth_zone** z)
 477 {
 478         struct auth_zone key;
 479         key.node.key = &key;
 480         key.dclass = dclass;
 481         key.name = nm;
 482         key.namelen = nmlen;
 483         key.namelabs = dname_count_labels(nm);
 484         return rbtree_find_less_equal(&az->ztree, &key, (rbnode_type**)z);
 485 }
 486
 487
 488 /** find the auth zone that is above the given name */
 489 struct auth_zone*
 490 auth_zones_find_zone(struct auth_zones* az, uint8_t* name, size_t name_len,
 491         uint16_t dclass)
 492 {
 493         uint8_t* nm = name;
 494         size_t nmlen = name_len;
 495         struct auth_zone* z;
 496         if(auth_zone_find_less_equal(az, nm, nmlen, dclass, &z)) {
 497                 /* exact match */
 498                 return z;
 499         } else {
 500                 /* less-or-nothing */
 501                 if(!z) return NULL; /* nothing smaller, nothing above it */
 502                 /* we found smaller name; smaller may be above the name,
 503                  * but not below it. */
 504                 nm = dname_get_shared_topdomain(z->name, name);
 505                 dname_count_size_labels(nm, &nmlen);
 506                 z = NULL;
 507         }
 508
 509         /* search up */
 510         while(!z) {
 511                 z = auth_zone_find(az, nm, nmlen, dclass);
 512                 if(z) return z;
 513                 if(dname_is_root(nm)) break;
 514                 dname_remove_label(&nm, &nmlen);
 515         }
 516         return NULL;
 517 }
 518
 519 /** find or create zone with name str. caller must have lock on az.
 520  * returns a wrlocked zone */
 521 static struct auth_zone*
 522 auth_zones_find_or_add_zone(struct auth_zones* az, char* name)
 523 {
 524         uint8_t nm[LDNS_MAX_DOMAINLEN+1];
 525         size_t nmlen = sizeof(nm);
 526         struct auth_zone* z;
 527
 528         if(sldns_str2wire_dname_buf(name, nm, &nmlen) != 0) {
 529                 log_err("cannot parse auth zone name: %s", name);
 530                 return 0;
 531         }
 532         z = auth_zone_find(az, nm, nmlen, LDNS_RR_CLASS_IN);
 533         if(!z) {
 534                 /* not found, create the zone */
 535                 z = auth_zone_create(az, nm, nmlen, LDNS_RR_CLASS_IN);
 536         } else {
 537                 lock_rw_wrlock(&z->lock);
 538         }
 539         return z;
 540 }
 541
 542 /** find or create xfer zone with name str. caller must have lock on az.
 543  * returns a locked xfer */
 544 static struct auth_xfer*
 545 auth_zones_find_or_add_xfer(struct auth_zones* az, struct auth_zone* z)
 546 {
 547         struct auth_xfer* x;
 548         x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
 549         if(!x) {
 550                 /* not found, create the zone */
 551                 x = auth_xfer_create(az, z);
 552         } else {
 553                 lock_basic_lock(&x->lock);
 554         }
 555         return x;
 556 }
 557
 558 int
 559 auth_zone_set_zonefile(struct auth_zone* z, char* zonefile)
 560 {
 561         if(z->zonefile) free(z->zonefile);
 562         if(zonefile == NULL) {
 563                 z->zonefile = NULL;
 564         } else {
 565                 z->zonefile = strdup(zonefile);
 566                 if(!z->zonefile) {
 567                         log_err("malloc failure");
 568                         return 0;
 569                 }
 570         }
 571         return 1;
 572 }
 573
 574 /** set auth zone fallback. caller must have lock on zone */
 575 int
 576 auth_zone_set_fallback(struct auth_zone* z, char* fallbackstr)
 577 {
 578         if(strcmp(fallbackstr, "yes") != 0 && strcmp(fallbackstr, "no") != 0){
 579                 log_err("auth zone fallback, expected yes or no, got %s",
 580                         fallbackstr);
 581                 return 0;
 582         }
 583         z->fallback_enabled = (strcmp(fallbackstr, "yes")==0);
 584         return 1;
 585 }
 586
 587 /** create domain with the given name */
 588 static struct auth_data*
 589 az_domain_create(struct auth_zone* z, uint8_t* nm, size_t nmlen)
 590 {
 591         struct auth_data* n = (struct auth_data*)malloc(sizeof(*n));
 592         if(!n) return NULL;
 593         memset(n, 0, sizeof(*n));
 594         n->node.key = n;
 595         n->name = memdup(nm, nmlen);
 596         if(!n->name) {
 597                 free(n);
 598                 return NULL;
 599         }
 600         n->namelen = nmlen;
 601         n->namelabs = dname_count_labels(nm);
 602         if(!rbtree_insert(&z->data, &n->node)) {
 603                 log_warn("duplicate auth domain name");
 604                 free(n->name);
 605                 free(n);
 606                 return NULL;
 607         }
 608         return n;
 609 }
 610
 611 /** find domain with exactly the given name */
 612 static struct auth_data*
 613 az_find_name(struct auth_zone* z, uint8_t* nm, size_t nmlen)
 614 {
 615         struct auth_zone key;
 616         key.node.key = &key;
 617         key.name = nm;
 618         key.namelen = nmlen;
 619         key.namelabs = dname_count_labels(nm);
 620         return (struct auth_data*)rbtree_search(&z->data, &key);
 621 }
 622
 623 /** Find domain name (or closest match) */
 624 static void
 625 az_find_domain(struct auth_zone* z, struct query_info* qinfo, int* node_exact,
 626         struct auth_data** node)
 627 {
 628         struct auth_zone key;
 629         key.node.key = &key;
 630         key.name = qinfo->qname;
 631         key.namelen = qinfo->qname_len;
 632         key.namelabs = dname_count_labels(key.name);
 633         *node_exact = rbtree_find_less_equal(&z->data, &key,
 634                 (rbnode_type**)node);
 635 }
 636
 637 /** find or create domain with name in zone */
 638 static struct auth_data*
 639 az_domain_find_or_create(struct auth_zone* z, uint8_t* dname,
 640         size_t dname_len)
 641 {
 642         struct auth_data* n = az_find_name(z, dname, dname_len);
 643         if(!n) {
 644                 n = az_domain_create(z, dname, dname_len);
 645         }
 646         return n;
 647 }
 648
 649 /** find rrset of given type in the domain */
 650 static struct auth_rrset*
 651 az_domain_rrset(struct auth_data* n, uint16_t t)
 652 {
 653         struct auth_rrset* rrset;
 654         if(!n) return NULL;
 655         rrset = n->rrsets;
 656         while(rrset) {
 657                 if(rrset->type == t)
 658                         return rrset;
 659                 rrset = rrset->next;
 660         }
 661         return NULL;
 662 }
 663
 664 /** remove rrset of this type from domain */
 665 static void
 666 domain_remove_rrset(struct auth_data* node, uint16_t rr_type)
 667 {
 668         struct auth_rrset* rrset, *prev;
 669         if(!node) return;
 670         prev = NULL;
 671         rrset = node->rrsets;
 672         while(rrset) {
 673                 if(rrset->type == rr_type) {
 674                         /* found it, now delete it */
 675                         if(prev) prev->next = rrset->next;
 676                         else    node->rrsets = rrset->next;
 677                         auth_rrset_delete(rrset);
 678                         return;
 679                 }
 680                 prev = rrset;
 681                 rrset = rrset->next;
 682         }
 683 }
 684
 685 /** find an rrsig index in the rrset.  returns true if found */
 686 static int
 687 az_rrset_find_rrsig(struct packed_rrset_data* d, uint8_t* rdata, size_t len,
 688         size_t* index)
 689 {
 690         size_t i;
 691         for(i=d->count; i<d->count + d->rrsig_count; i++) {
 692                 if(d->rr_len[i] != len)
 693                         continue;
 694                 if(memcmp(d->rr_data[i], rdata, len) == 0) {
 695                         *index = i;
 696                         return 1;
 697                 }
 698         }
 699         return 0;
 700 }
 701
 702 /** see if rdata is duplicate */
 703 static int
 704 rdata_duplicate(struct packed_rrset_data* d, uint8_t* rdata, size_t len)
 705 {
 706         size_t i;
 707         for(i=0; i<d->count + d->rrsig_count; i++) {
 708                 if(d->rr_len[i] != len)
 709                         continue;
 710                 if(memcmp(d->rr_data[i], rdata, len) == 0)
 711                         return 1;
 712         }
 713         return 0;
 714 }
 715
 716 /** get rrsig type covered from rdata.
 717  * @param rdata: rdata in wireformat, starting with 16bit rdlength.
 718  * @param rdatalen: length of rdata buffer.
 719  * @return type covered (or 0).
 720  */
 721 static uint16_t
 722 rrsig_rdata_get_type_covered(uint8_t* rdata, size_t rdatalen)
 723 {
 724         if(rdatalen < 4)
 725                 return 0;
 726         return sldns_read_uint16(rdata+2);
 727 }
 728
 729 /** remove RR from existing RRset. Also sig, if it is a signature.
 730  * reallocates the packed rrset for a new one, false on alloc failure */
 731 static int
 732 rrset_remove_rr(struct auth_rrset* rrset, size_t index)
 733 {
 734         struct packed_rrset_data* d, *old = rrset->data;
 735         size_t i;
 736         if(index >= old->count + old->rrsig_count)
 737                 return 0; /* index out of bounds */
 738         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old) - (
 739                 sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t) +
 740                 old->rr_len[index]));
 741         if(!d) {
 742                 log_err("malloc failure");
 743                 return 0;
 744         }
 745         d->ttl = old->ttl;
 746         d->count = old->count;
 747         d->rrsig_count = old->rrsig_count;
 748         if(index < d->count) d->count--;
 749         else d->rrsig_count--;
 750         d->trust = old->trust;
 751         d->security = old->security;
 752
 753         /* set rr_len, needed for ptr_fixup */
 754         d->rr_len = (size_t*)((uint8_t*)d +
 755                 sizeof(struct packed_rrset_data));
 756         if(index > 0)
 757                 memmove(d->rr_len, old->rr_len, (index)*sizeof(size_t));
 758         if(index+1 < old->count+old->rrsig_count)
 759                 memmove(&d->rr_len[index], &old->rr_len[index+1],
 760                 (old->count+old->rrsig_count - (index+1))*sizeof(size_t));
 761         packed_rrset_ptr_fixup(d);
 762
 763         /* move over ttls */
 764         if(index > 0)
 765                 memmove(d->rr_ttl, old->rr_ttl, (index)*sizeof(time_t));
 766         if(index+1 < old->count+old->rrsig_count)
 767                 memmove(&d->rr_ttl[index], &old->rr_ttl[index+1],
 768                 (old->count+old->rrsig_count - (index+1))*sizeof(time_t));
 769
 770         /* move over rr_data */
 771         for(i=0; i<d->count+d->rrsig_count; i++) {
 772                 size_t oldi;
 773                 if(i < index) oldi = i;
 774                 else oldi = i+1;
 775                 memmove(d->rr_data[i], old->rr_data[oldi], d->rr_len[i]);
 776         }
 777
 778         /* recalc ttl (lowest of remaining RR ttls) */
 779         if(d->count + d->rrsig_count > 0)
 780                 d->ttl = d->rr_ttl[0];
 781         for(i=0; i<d->count+d->rrsig_count; i++) {
 782                 if(d->rr_ttl[i] < d->ttl)
 783                         d->ttl = d->rr_ttl[i];
 784         }
 785
 786         free(rrset->data);
 787         rrset->data = d;
 788         return 1;
 789 }
 790
 791 /** add RR to existing RRset. If insert_sig is true, add to rrsigs.
 792  * This reallocates the packed rrset for a new one */
 793 static int
 794 rrset_add_rr(struct auth_rrset* rrset, uint32_t rr_ttl, uint8_t* rdata,
 795         size_t rdatalen, int insert_sig)
 796 {
 797         struct packed_rrset_data* d, *old = rrset->data;
 798         size_t total, old_total;
 799
 800         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
 801                 + sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t)
 802                 + rdatalen);
 803         if(!d) {
 804                 log_err("out of memory");
 805                 return 0;
 806         }
 807         /* copy base values */
 808         memcpy(d, old, sizeof(struct packed_rrset_data));
 809         if(!insert_sig) {
 810                 d->count++;
 811         } else {
 812                 d->rrsig_count++;
 813         }
 814         old_total = old->count + old->rrsig_count;
 815         total = d->count + d->rrsig_count;
 816         /* set rr_len, needed for ptr_fixup */
 817         d->rr_len = (size_t*)((uint8_t*)d +
 818                 sizeof(struct packed_rrset_data));
 819         if(old->count != 0)
 820                 memmove(d->rr_len, old->rr_len, old->count*sizeof(size_t));
 821         if(old->rrsig_count != 0)
 822                 memmove(d->rr_len+d->count, old->rr_len+old->count,
 823                         old->rrsig_count*sizeof(size_t));
 824         if(!insert_sig)
 825                 d->rr_len[d->count-1] = rdatalen;
 826         else    d->rr_len[total-1] = rdatalen;
 827         packed_rrset_ptr_fixup(d);
 828         if((time_t)rr_ttl < d->ttl)
 829                 d->ttl = rr_ttl;
 830
 831         /* copy old values into new array */
 832         if(old->count != 0) {
 833                 memmove(d->rr_ttl, old->rr_ttl, old->count*sizeof(time_t));
 834                 /* all the old rr pieces are allocated sequential, so we
 835                  * can copy them in one go */
 836                 memmove(d->rr_data[0], old->rr_data[0],
 837                         (old->rr_data[old->count-1] - old->rr_data[0]) +
 838                         old->rr_len[old->count-1]);
 839         }
 840         if(old->rrsig_count != 0) {
 841                 memmove(d->rr_ttl+d->count, old->rr_ttl+old->count,
 842                         old->rrsig_count*sizeof(time_t));
 843                 memmove(d->rr_data[d->count], old->rr_data[old->count],
 844                         (old->rr_data[old_total-1] - old->rr_data[old->count]) +
 845                         old->rr_len[old_total-1]);
 846         }
 847
 848         /* insert new value */
 849         if(!insert_sig) {
 850                 d->rr_ttl[d->count-1] = rr_ttl;
 851                 memmove(d->rr_data[d->count-1], rdata, rdatalen);
 852         } else {
 853                 d->rr_ttl[total-1] = rr_ttl;
 854                 memmove(d->rr_data[total-1], rdata, rdatalen);
 855         }
 856
 857         rrset->data = d;
 858         free(old);
 859         return 1;
 860 }
 861
 862 /** Create new rrset for node with packed rrset with one RR element */
 863 static struct auth_rrset*
 864 rrset_create(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
 865         uint8_t* rdata, size_t rdatalen)
 866 {
 867         struct auth_rrset* rrset = (struct auth_rrset*)calloc(1,
 868                 sizeof(*rrset));
 869         struct auth_rrset* p, *prev;
 870         struct packed_rrset_data* d;
 871         if(!rrset) {
 872                 log_err("out of memory");
 873                 return NULL;
 874         }
 875         rrset->type = rr_type;
 876
 877         /* the rrset data structure, with one RR */
 878         d = (struct packed_rrset_data*)calloc(1,
 879                 sizeof(struct packed_rrset_data) + sizeof(size_t) +
 880                 sizeof(uint8_t*) + sizeof(time_t) + rdatalen);
 881         if(!d) {
 882                 free(rrset);
 883                 log_err("out of memory");
 884                 return NULL;
 885         }
 886         rrset->data = d;
 887         d->ttl = rr_ttl;
 888         d->trust = rrset_trust_prim_noglue;
 889         d->rr_len = (size_t*)((uint8_t*)d + sizeof(struct packed_rrset_data));
 890         d->rr_data = (uint8_t**)&(d->rr_len[1]);
 891         d->rr_ttl = (time_t*)&(d->rr_data[1]);
 892         d->rr_data[0] = (uint8_t*)&(d->rr_ttl[1]);
 893
 894         /* insert the RR */
 895         d->rr_len[0] = rdatalen;
 896         d->rr_ttl[0] = rr_ttl;
 897         memmove(d->rr_data[0], rdata, rdatalen);
 898         d->count++;
 899
 900         /* insert rrset into linked list for domain */
 901         /* find sorted place to link the rrset into the list */
 902         prev = NULL;
 903         p = node->rrsets;
 904         while(p && p->type<=rr_type) {
 905                 prev = p;
 906                 p = p->next;
 907         }
 908         /* so, prev is smaller, and p is larger than rr_type */
 909         rrset->next = p;
 910         if(prev) prev->next = rrset;
 911         else node->rrsets = rrset;
 912         return rrset;
 913 }
 914
 915 /** count number (and size) of rrsigs that cover a type */
 916 static size_t
 917 rrsig_num_that_cover(struct auth_rrset* rrsig, uint16_t rr_type, size_t* sigsz)
 918 {
 919         struct packed_rrset_data* d = rrsig->data;
 920         size_t i, num = 0;
 921         *sigsz = 0;
 922         log_assert(d && rrsig->type == LDNS_RR_TYPE_RRSIG);
 923         for(i=0; i<d->count+d->rrsig_count; i++) {
 924                 if(rrsig_rdata_get_type_covered(d->rr_data[i],
 925                         d->rr_len[i]) == rr_type) {
 926                         num++;
 927                         (*sigsz) += d->rr_len[i];
 928                 }
 929         }
 930         return num;
 931 }
 932
 933 /** See if rrsig set has covered sigs for rrset and move them over */
 934 static int
 935 rrset_moveover_rrsigs(struct auth_data* node, uint16_t rr_type,
 936         struct auth_rrset* rrset, struct auth_rrset* rrsig)
 937 {
 938         size_t sigs, sigsz, i, j, total;
 939         struct packed_rrset_data* sigold = rrsig->data;
 940         struct packed_rrset_data* old = rrset->data;
 941         struct packed_rrset_data* d, *sigd;
 942
 943         log_assert(rrset->type == rr_type);
 944         log_assert(rrsig->type == LDNS_RR_TYPE_RRSIG);
 945         sigs = rrsig_num_that_cover(rrsig, rr_type, &sigsz);
 946         if(sigs == 0) {
 947                 /* 0 rrsigs to move over, done */
 948                 return 1;
 949         }
 950
 951         /* allocate rrset sigsz larger for extra sigs elements, and
 952          * allocate rrsig sigsz smaller for less sigs elements. */
 953         d = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(old)
 954                 + sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
 955                 + sigsz);
 956         if(!d) {
 957                 log_err("out of memory");
 958                 return 0;
 959         }
 960         /* copy base values */
 961         total = old->count + old->rrsig_count;
 962         memcpy(d, old, sizeof(struct packed_rrset_data));
 963         d->rrsig_count += sigs;
 964         /* setup rr_len */
 965         d->rr_len = (size_t*)((uint8_t*)d +
 966                 sizeof(struct packed_rrset_data));
 967         if(total != 0)
 968                 memmove(d->rr_len, old->rr_len, total*sizeof(size_t));
 969         j = d->count+d->rrsig_count-sigs;
 970         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
 971                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
 972                         sigold->rr_len[i]) == rr_type) {
 973                         d->rr_len[j] = sigold->rr_len[i];
 974                         j++;
 975                 }
 976         }
 977         packed_rrset_ptr_fixup(d);
 978
 979         /* copy old values into new array */
 980         if(total != 0) {
 981                 memmove(d->rr_ttl, old->rr_ttl, total*sizeof(time_t));
 982                 /* all the old rr pieces are allocated sequential, so we
 983                  * can copy them in one go */
 984                 memmove(d->rr_data[0], old->rr_data[0],
 985                         (old->rr_data[total-1] - old->rr_data[0]) +
 986                         old->rr_len[total-1]);
 987         }
 988
 989         /* move over the rrsigs to the larger rrset*/
 990         j = d->count+d->rrsig_count-sigs;
 991         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
 992                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
 993                         sigold->rr_len[i]) == rr_type) {
 994                         /* move this one over to location j */
 995                         d->rr_ttl[j] = sigold->rr_ttl[i];
 996                         memmove(d->rr_data[j], sigold->rr_data[i],
 997                                 sigold->rr_len[i]);
 998                         if(d->rr_ttl[j] < d->ttl)
 999                                 d->ttl = d->rr_ttl[j];
1000                         j++;
1001                 }
1002         }
1003
1004         /* put it in and deallocate the old rrset */
1005         rrset->data = d;
1006         free(old);
1007
1008         /* now make rrsig set smaller */
1009         if(sigold->count+sigold->rrsig_count == sigs) {
1010                 /* remove all sigs from rrsig, remove it entirely */
1011                 domain_remove_rrset(node, LDNS_RR_TYPE_RRSIG);
1012                 return 1;
1013         }
1014         log_assert(packed_rrset_sizeof(sigold) > sigs*(sizeof(size_t) +
1015                 sizeof(uint8_t*) + sizeof(time_t)) + sigsz);
1016         sigd = (struct packed_rrset_data*)calloc(1, packed_rrset_sizeof(sigold)
1017                 - sigs*(sizeof(size_t) + sizeof(uint8_t*) + sizeof(time_t))
1018                 - sigsz);
1019         if(!sigd) {
1020                 /* no need to free up d, it has already been placed in the
1021                  * node->rrset structure */
1022                 log_err("out of memory");
1023                 return 0;
1024         }
1025         /* copy base values */
1026         memcpy(sigd, sigold, sizeof(struct packed_rrset_data));
1027         /* in sigd the RRSIGs are stored in the base of the RR, in count */
1028         sigd->count -= sigs;
1029         /* setup rr_len */
1030         sigd->rr_len = (size_t*)((uint8_t*)sigd +
1031                 sizeof(struct packed_rrset_data));
1032         j = 0;
1033         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1034                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1035                         sigold->rr_len[i]) != rr_type) {
1036                         sigd->rr_len[j] = sigold->rr_len[i];
1037                         j++;
1038                 }
1039         }
1040         packed_rrset_ptr_fixup(sigd);
1041
1042         /* copy old values into new rrsig array */
1043         j = 0;
1044         for(i=0; i<sigold->count+sigold->rrsig_count; i++) {
1045                 if(rrsig_rdata_get_type_covered(sigold->rr_data[i],
1046                         sigold->rr_len[i]) != rr_type) {
1047                         /* move this one over to location j */
1048                         sigd->rr_ttl[j] = sigold->rr_ttl[i];
1049                         memmove(sigd->rr_data[j], sigold->rr_data[i],
1050                                 sigold->rr_len[i]);
1051                         if(j==0) sigd->ttl = sigd->rr_ttl[j];
1052                         else {
1053                                 if(sigd->rr_ttl[j] < sigd->ttl)
1054                                         sigd->ttl = sigd->rr_ttl[j];
1055                         }
1056                         j++;
1057                 }
1058         }
1059
1060         /* put it in and deallocate the old rrset */
1061         rrsig->data = sigd;
1062         free(sigold);
1063
1064         return 1;
1065 }
1066
1067 /** copy the rrsigs from the rrset to the rrsig rrset, because the rrset
1068  * is going to be deleted.  reallocates the RRSIG rrset data. */
1069 static int
1070 rrsigs_copy_from_rrset_to_rrsigset(struct auth_rrset* rrset,
1071         struct auth_rrset* rrsigset)
1072 {
1073         size_t i;
1074         if(rrset->data->rrsig_count == 0)
1075                 return 1;
1076
1077         /* move them over one by one, because there might be duplicates,
1078          * duplicates are ignored */
1079         for(i=rrset->data->count;
1080                 i<rrset->data->count+rrset->data->rrsig_count; i++) {
1081                 uint8_t* rdata = rrset->data->rr_data[i];
1082                 size_t rdatalen = rrset->data->rr_len[i];
1083                 time_t rr_ttl  = rrset->data->rr_ttl[i];
1084
1085                 if(rdata_duplicate(rrsigset->data, rdata, rdatalen)) {
1086                         continue;
1087                 }
1088                 if(!rrset_add_rr(rrsigset, rr_ttl, rdata, rdatalen, 0))
1089                         return 0;
1090         }
1091         return 1;
1092 }
1093
1094 /** Add rr to node, ignores duplicate RRs,
1095  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1096 static int
1097 az_domain_add_rr(struct auth_data* node, uint16_t rr_type, uint32_t rr_ttl,
1098         uint8_t* rdata, size_t rdatalen, int* duplicate)
1099 {
1100         struct auth_rrset* rrset;
1101         /* packed rrsets have their rrsigs along with them, sort them out */
1102         if(rr_type == LDNS_RR_TYPE_RRSIG) {
1103                 uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1104                 if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1105                         /* a node of the correct type exists, add the RRSIG
1106                          * to the rrset of the covered data type */
1107                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1108                                 if(duplicate) *duplicate = 1;
1109                                 return 1;
1110                         }
1111                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 1))
1112                                 return 0;
1113                 } else if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1114                         /* add RRSIG to rrset of type RRSIG */
1115                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1116                                 if(duplicate) *duplicate = 1;
1117                                 return 1;
1118                         }
1119                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1120                                 return 0;
1121                 } else {
1122                         /* create rrset of type RRSIG */
1123                         if(!rrset_create(node, rr_type, rr_ttl, rdata,
1124                                 rdatalen))
1125                                 return 0;
1126                 }
1127         } else {
1128                 /* normal RR type */
1129                 if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1130                         /* add data to existing node with data type */
1131                         if(rdata_duplicate(rrset->data, rdata, rdatalen)) {
1132                                 if(duplicate) *duplicate = 1;
1133                                 return 1;
1134                         }
1135                         if(!rrset_add_rr(rrset, rr_ttl, rdata, rdatalen, 0))
1136                                 return 0;
1137                 } else {
1138                         struct auth_rrset* rrsig;
1139                         /* create new node with data type */
1140                         if(!(rrset=rrset_create(node, rr_type, rr_ttl, rdata,
1141                                 rdatalen)))
1142                                 return 0;
1143
1144                         /* see if node of type RRSIG has signatures that
1145                          * cover the data type, and move them over */
1146                         /* and then make the RRSIG type smaller */
1147                         if((rrsig=az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
1148                                 != NULL) {
1149                                 if(!rrset_moveover_rrsigs(node, rr_type,
1150                                         rrset, rrsig))
1151                                         return 0;
1152                         }
1153                 }
1154         }
1155         return 1;
1156 }
1157
1158 /** insert RR into zone, ignore duplicates */
1159 static int
1160 az_insert_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1161         size_t dname_len, int* duplicate)
1162 {
1163         struct auth_data* node;
1164         uint8_t* dname = rr;
1165         uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1166         uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1167         uint32_t rr_ttl = sldns_wirerr_get_ttl(rr, rr_len, dname_len);
1168         size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1169                 dname_len))+2;
1170         /* rdata points to rdata prefixed with uint16 rdatalength */
1171         uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1172
1173         if(rr_class != z->dclass) {
1174                 log_err("wrong class for RR");
1175                 return 0;
1176         }
1177         if(!(node=az_domain_find_or_create(z, dname, dname_len))) {
1178                 log_err("cannot create domain");
1179                 return 0;
1180         }
1181         if(!az_domain_add_rr(node, rr_type, rr_ttl, rdata, rdatalen,
1182                 duplicate)) {
1183                 log_err("cannot add RR to domain");
1184                 return 0;
1185         }
1186         if(z->rpz) {
1187                 if(!(rpz_insert_rr(z->rpz, z->name, z->namelen, dname,
1188                         dname_len, rr_type, rr_class, rr_ttl, rdata, rdatalen,
1189                         rr, rr_len)))
1190                         return 0;
1191         }
1192         return 1;
1193 }
1194
1195 /** Remove rr from node, ignores nonexisting RRs,
1196  * rdata points to buffer with rdatalen octets, starts with 2bytelength. */
1197 static int
1198 az_domain_remove_rr(struct auth_data* node, uint16_t rr_type,
1199         uint8_t* rdata, size_t rdatalen, int* nonexist)
1200 {
1201         struct auth_rrset* rrset;
1202         size_t index = 0;
1203
1204         /* find the plain RR of the given type */
1205         if((rrset=az_domain_rrset(node, rr_type))!= NULL) {
1206                 if(packed_rrset_find_rr(rrset->data, rdata, rdatalen, &index)) {
1207                         if(rrset->data->count == 1 &&
1208                                 rrset->data->rrsig_count == 0) {
1209                                 /* last RR, delete the rrset */
1210                                 domain_remove_rrset(node, rr_type);
1211                         } else if(rrset->data->count == 1 &&
1212                                 rrset->data->rrsig_count != 0) {
1213                                 /* move RRSIGs to the RRSIG rrset, or
1214                                  * this one becomes that RRset */
1215                                 struct auth_rrset* rrsigset = az_domain_rrset(
1216                                         node, LDNS_RR_TYPE_RRSIG);
1217                                 if(rrsigset) {
1218                                         /* move left over rrsigs to the
1219                                          * existing rrset of type RRSIG */
1220                                         rrsigs_copy_from_rrset_to_rrsigset(
1221                                                 rrset, rrsigset);
1222                                         /* and then delete the rrset */
1223                                         domain_remove_rrset(node, rr_type);
1224                                 } else {
1225                                         /* no rrset of type RRSIG, this
1226                                          * set is now of that type,
1227                                          * just remove the rr */
1228                                         if(!rrset_remove_rr(rrset, index))
1229                                                 return 0;
1230                                         rrset->type = LDNS_RR_TYPE_RRSIG;
1231                                         rrset->data->count = rrset->data->rrsig_count;
1232                                         rrset->data->rrsig_count = 0;
1233                                 }
1234                         } else {
1235                                 /* remove the RR from the rrset */
1236                                 if(!rrset_remove_rr(rrset, index))
1237                                         return 0;
1238                         }
1239                         return 1;
1240                 }
1241                 /* rr not found in rrset */
1242         }
1243
1244         /* is it a type RRSIG, look under the covered type */
1245         if(rr_type == LDNS_RR_TYPE_RRSIG) {
1246                 uint16_t ctype = rrsig_rdata_get_type_covered(rdata, rdatalen);
1247                 if((rrset=az_domain_rrset(node, ctype))!= NULL) {
1248                         if(az_rrset_find_rrsig(rrset->data, rdata, rdatalen,
1249                                 &index)) {
1250                                 /* rrsig should have d->count > 0, be
1251                                  * over some rr of that type */
1252                                 /* remove the rrsig from the rrsigs list of the
1253                                  * rrset */
1254                                 if(!rrset_remove_rr(rrset, index))
1255                                         return 0;
1256                                 return 1;
1257                         }
1258                 }
1259                 /* also RRSIG not found */
1260         }
1261
1262         /* nothing found to delete */
1263         if(nonexist) *nonexist = 1;
1264         return 1;
1265 }
1266
1267 /** remove RR from zone, ignore if it does not exist, false on alloc failure*/
1268 static int
1269 az_remove_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
1270         size_t dname_len, int* nonexist)
1271 {
1272         struct auth_data* node;
1273         uint8_t* dname = rr;
1274         uint16_t rr_type = sldns_wirerr_get_type(rr, rr_len, dname_len);
1275         uint16_t rr_class = sldns_wirerr_get_class(rr, rr_len, dname_len);
1276         size_t rdatalen = ((size_t)sldns_wirerr_get_rdatalen(rr, rr_len,
1277                 dname_len))+2;
1278         /* rdata points to rdata prefixed with uint16 rdatalength */
1279         uint8_t* rdata = sldns_wirerr_get_rdatawl(rr, rr_len, dname_len);
1280
1281         if(rr_class != z->dclass) {
1282                 log_err("wrong class for RR");
1283                 /* really also a nonexisting entry, because no records
1284                  * of that class in the zone, but return an error because
1285                  * getting records of the wrong class is a failure of the
1286                  * zone transfer */
1287                 return 0;
1288         }
1289         node = az_find_name(z, dname, dname_len);
1290         if(!node) {
1291                 /* node with that name does not exist */
1292                 /* nonexisting entry, because no such name */
1293                 *nonexist = 1;
1294                 return 1;
1295         }
1296         if(!az_domain_remove_rr(node, rr_type, rdata, rdatalen, nonexist)) {
1297                 /* alloc failure or so */
1298                 return 0;
1299         }
1300         /* remove the node, if necessary */
1301         /* an rrsets==NULL entry is not kept around for empty nonterminals,
1302          * and also parent nodes are not kept around, so we just delete it */
1303         if(node->rrsets == NULL) {
1304                 (void)rbtree_delete(&z->data, node);
1305                 auth_data_delete(node);
1306         }
1307         if(z->rpz) {
1308                 rpz_remove_rr(z->rpz, z->namelen, dname, dname_len, rr_type,
1309                         rr_class, rdata, rdatalen);
1310         }
1311         return 1;
1312 }
1313
1314 /** decompress an RR into the buffer where it'll be an uncompressed RR
1315  * with uncompressed dname and uncompressed rdata (dnames) */
1316 static int
1317 decompress_rr_into_buffer(struct sldns_buffer* buf, uint8_t* pkt,
1318         size_t pktlen, uint8_t* dname, uint16_t rr_type, uint16_t rr_class,
1319         uint32_t rr_ttl, uint8_t* rr_data, uint16_t rr_rdlen)
1320 {
1321         sldns_buffer pktbuf;
1322         size_t dname_len = 0;
1323         size_t rdlenpos;
1324         size_t rdlen;
1325         uint8_t* rd;
1326         const sldns_rr_descriptor* desc;
1327         sldns_buffer_init_frm_data(&pktbuf, pkt, pktlen);
1328         sldns_buffer_clear(buf);
1329
1330         /* decompress dname */
1331         sldns_buffer_set_position(&pktbuf,
1332                 (size_t)(dname - sldns_buffer_current(&pktbuf)));
1333         dname_len = pkt_dname_len(&pktbuf);
1334         if(dname_len == 0) return 0; /* parse fail on dname */
1335         if(!sldns_buffer_available(buf, dname_len)) return 0;
1336         dname_pkt_copy(&pktbuf, sldns_buffer_current(buf), dname);
1337         sldns_buffer_skip(buf, (ssize_t)dname_len);
1338
1339         /* type, class, ttl and rdatalength fields */
1340         if(!sldns_buffer_available(buf, 10)) return 0;
1341         sldns_buffer_write_u16(buf, rr_type);
1342         sldns_buffer_write_u16(buf, rr_class);
1343         sldns_buffer_write_u32(buf, rr_ttl);
1344         rdlenpos = sldns_buffer_position(buf);
1345         sldns_buffer_write_u16(buf, 0); /* rd length position */
1346
1347         /* decompress rdata */
1348         desc = sldns_rr_descript(rr_type);
1349         rd = rr_data;
1350         rdlen = rr_rdlen;
1351         if(rdlen > 0 && desc && desc->_dname_count > 0) {
1352                 int count = (int)desc->_dname_count;
1353                 int rdf = 0;
1354                 size_t len; /* how much rdata to plain copy */
1355                 size_t uncompressed_len, compressed_len;
1356                 size_t oldpos;
1357                 /* decompress dnames. */
1358                 while(rdlen > 0 && count) {
1359                         switch(desc->_wireformat[rdf]) {
1360                         case LDNS_RDF_TYPE_DNAME:
1361                                 sldns_buffer_set_position(&pktbuf,
1362                                         (size_t)(rd -
1363                                         sldns_buffer_begin(&pktbuf)));
1364                                 oldpos = sldns_buffer_position(&pktbuf);
1365                                 /* moves pktbuf to right after the
1366                                  * compressed dname, and returns uncompressed
1367                                  * dname length */
1368                                 uncompressed_len = pkt_dname_len(&pktbuf);
1369                                 if(!uncompressed_len)
1370                                         return 0; /* parse error in dname */
1371                                 if(!sldns_buffer_available(buf,
1372                                         uncompressed_len))
1373                                         /* dname too long for buffer */
1374                                         return 0;
1375                                 dname_pkt_copy(&pktbuf,
1376                                         sldns_buffer_current(buf), rd);
1377                                 sldns_buffer_skip(buf, (ssize_t)uncompressed_len);
1378                                 compressed_len = sldns_buffer_position(
1379                                         &pktbuf) - oldpos;
1380                                 rd += compressed_len;
1381                                 rdlen -= compressed_len;
1382                                 count--;
1383                                 len = 0;
1384                                 break;
1385                         case LDNS_RDF_TYPE_STR:
1386                                 len = rd[0] + 1;
1387                                 break;
1388                         default:
1389                                 len = get_rdf_size(desc->_wireformat[rdf]);
1390                                 break;
1391                         }
1392                         if(len) {
1393                                 if(!sldns_buffer_available(buf, len))
1394                                         return 0; /* too long for buffer */
1395                                 sldns_buffer_write(buf, rd, len);
1396                                 rd += len;
1397                                 rdlen -= len;
1398                         }
1399                         rdf++;
1400                 }
1401         }
1402         /* copy remaining data */
1403         if(rdlen > 0) {
1404                 if(!sldns_buffer_available(buf, rdlen)) return 0;
1405                 sldns_buffer_write(buf, rd, rdlen);
1406         }
1407         /* fixup rdlength */
1408         sldns_buffer_write_u16_at(buf, rdlenpos,
1409                 sldns_buffer_position(buf)-rdlenpos-2);
1410         sldns_buffer_flip(buf);
1411         return 1;
1412 }
1413
1414 /** insert RR into zone, from packet, decompress RR,
1415  * if duplicate is nonNULL set the flag but otherwise ignore duplicates */
1416 static int
1417 az_insert_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1418         struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1419         uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1420         uint16_t rr_rdlen, int* duplicate)
1421 {
1422         uint8_t* rr;
1423         size_t rr_len;
1424         size_t dname_len;
1425         if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1426                 rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1427                 log_err("could not decompress RR");
1428                 return 0;
1429         }
1430         rr = sldns_buffer_begin(scratch_buffer);
1431         rr_len = sldns_buffer_limit(scratch_buffer);
1432         dname_len = dname_valid(rr, rr_len);
1433         return az_insert_rr(z, rr, rr_len, dname_len, duplicate);
1434 }
1435
1436 /** remove RR from zone, from packet, decompress RR,
1437  * if nonexist is nonNULL set the flag but otherwise ignore nonexisting entries*/
1438 static int
1439 az_remove_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
1440         struct sldns_buffer* scratch_buffer, uint8_t* dname, uint16_t rr_type,
1441         uint16_t rr_class, uint32_t rr_ttl, uint8_t* rr_data,
1442         uint16_t rr_rdlen, int* nonexist)
1443 {
1444         uint8_t* rr;
1445         size_t rr_len;
1446         size_t dname_len;
1447         if(!decompress_rr_into_buffer(scratch_buffer, pkt, pktlen, dname,
1448                 rr_type, rr_class, rr_ttl, rr_data, rr_rdlen)) {
1449                 log_err("could not decompress RR");
1450                 return 0;
1451         }
1452         rr = sldns_buffer_begin(scratch_buffer);
1453         rr_len = sldns_buffer_limit(scratch_buffer);
1454         dname_len = dname_valid(rr, rr_len);
1455         return az_remove_rr(z, rr, rr_len, dname_len, nonexist);
1456 }
1457
1458 /**
1459  * Parse zonefile
1460  * @param z: zone to read in.
1461  * @param in: file to read from (just opened).
1462  * @param rr: buffer to use for RRs, 64k.
1463  *      passed so that recursive includes can use the same buffer and do
1464  *      not grow the stack too much.
1465  * @param rrbuflen: sizeof rr buffer.
1466  * @param state: parse state with $ORIGIN, $TTL and 'prev-dname' and so on,
1467  *      that is kept between includes.
1468  *      The lineno is set at 1 and then increased by the function.
1469  * @param fname: file name.
1470  * @param depth: recursion depth for includes
1471  * @param cfg: config for chroot.
1472  * returns false on failure, has printed an error message
1473  */
1474 static int
1475 az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
1476         struct sldns_file_parse_state* state, char* fname, int depth,
1477         struct config_file* cfg)
1478 {
1479         size_t rr_len, dname_len;
1480         int status;
1481         state->lineno = 1;
1482
1483         while(!feof(in)) {
1484                 rr_len = rrbuflen;
1485                 dname_len = 0;
1486                 status = sldns_fp2wire_rr_buf(in, rr, &rr_len, &dname_len,
1487                         state);
1488                 if(status == LDNS_WIREPARSE_ERR_INCLUDE && rr_len == 0) {
1489                         /* we have $INCLUDE or $something */
1490                         if(strncmp((char*)rr, "$INCLUDE ", 9) == 0 ||
1491                            strncmp((char*)rr, "$INCLUDE\t", 9) == 0) {
1492                                 FILE* inc;
1493                                 int lineno_orig = state->lineno;
1494                                 char* incfile = (char*)rr + 8;
1495                                 if(depth > MAX_INCLUDE_DEPTH) {
1496                                         log_err("%s:%d max include depth"
1497                                           "exceeded", fname, state->lineno);
1498                                         return 0;
1499                                 }
1500                                 /* skip spaces */
1501                                 while(*incfile == ' ' || *incfile == '\t')
1502                                         incfile++;
1503                                 /* adjust for chroot on include file */
1504                                 if(cfg->chrootdir && cfg->chrootdir[0] &&
1505                                         strncmp(incfile, cfg->chrootdir,
1506                                                 strlen(cfg->chrootdir)) == 0)
1507                                         incfile += strlen(cfg->chrootdir);
1508                                 incfile = strdup(incfile);
1509                                 if(!incfile) {
1510                                         log_err("malloc failure");
1511                                         return 0;
1512                                 }
1513                                 verbose(VERB_ALGO, "opening $INCLUDE %s",
1514                                         incfile);
1515                                 inc = fopen(incfile, "r");
1516                                 if(!inc) {
1517                                         log_err("%s:%d cannot open include "
1518                                                 "file %s: %s", fname,
1519                                                 lineno_orig, incfile,
1520                                                 strerror(errno));
1521                                         free(incfile);
1522                                         return 0;
1523                                 }
1524                                 /* recurse read that file now */
1525                                 if(!az_parse_file(z, inc, rr, rrbuflen,
1526                                         state, incfile, depth+1, cfg)) {
1527                                         log_err("%s:%d cannot parse include "
1528                                                 "file %s", fname,
1529                                                 lineno_orig, incfile);
1530                                         fclose(inc);
1531                                         free(incfile);
1532                                         return 0;
1533                                 }
1534                                 fclose(inc);
1535                                 verbose(VERB_ALGO, "done with $INCLUDE %s",
1536                                         incfile);
1537                                 free(incfile);
1538                                 state->lineno = lineno_orig;
1539                         }
1540                         continue;
1541                 }
1542                 if(status != 0) {
1543                         log_err("parse error %s %d:%d: %s", fname,
1544                                 state->lineno, LDNS_WIREPARSE_OFFSET(status),
1545                                 sldns_get_errorstr_parse(status));
1546                         return 0;
1547                 }
1548                 if(rr_len == 0) {
1549                         /* EMPTY line, TTL or ORIGIN */
1550                         continue;
1551                 }
1552                 /* insert wirerr in rrbuf */
1553                 if(!az_insert_rr(z, rr, rr_len, dname_len, NULL)) {
1554                         char buf[17];
1555                         sldns_wire2str_type_buf(sldns_wirerr_get_type(rr,
1556                                 rr_len, dname_len), buf, sizeof(buf));
1557                         log_err("%s:%d cannot insert RR of type %s",
1558                                 fname, state->lineno, buf);
1559                         return 0;
1560                 }
1561         }
1562         return 1;
1563 }
1564
1565 int
1566 auth_zone_read_zonefile(struct auth_zone* z, struct config_file* cfg)
1567 {
1568         uint8_t rr[LDNS_RR_BUF_SIZE];
1569         struct sldns_file_parse_state state;
1570         char* zfilename;
1571         FILE* in;
1572         if(!z || !z->zonefile || z->zonefile[0]==0)
1573                 return 1; /* no file, or "", nothing to read */
1574
1575         zfilename = z->zonefile;
1576         if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
1577                 cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
1578                 zfilename += strlen(cfg->chrootdir);
1579         if(verbosity >= VERB_ALGO) {
1580                 char nm[255+1];
1581                 dname_str(z->name, nm);
1582                 verbose(VERB_ALGO, "read zonefile %s for %s", zfilename, nm);
1583         }
1584         in = fopen(zfilename, "r");
1585         if(!in) {
1586                 char* n = sldns_wire2str_dname(z->name, z->namelen);
1587                 if(z->zone_is_slave && errno == ENOENT) {
1588                         /* we fetch the zone contents later, no file yet */
1589                         verbose(VERB_ALGO, "no zonefile %s for %s",
1590                                 zfilename, n?n:"error");
1591                         free(n);
1592                         return 1;
1593                 }
1594                 log_err("cannot open zonefile %s for %s: %s",
1595                         zfilename, n?n:"error", strerror(errno));
1596                 free(n);
1597                 return 0;
1598         }
1599
1600         /* clear the data tree */
1601         traverse_postorder(&z->data, auth_data_del, NULL);
1602         rbtree_init(&z->data, &auth_data_cmp);
1603         /* clear the RPZ policies */
1604         if(z->rpz)
1605                 rpz_clear(z->rpz);
1606
1607         memset(&state, 0, sizeof(state));
1608         /* default TTL to 3600 */
1609         state.default_ttl = 3600;
1610         /* set $ORIGIN to the zone name */
1611         if(z->namelen <= sizeof(state.origin)) {
1612                 memcpy(state.origin, z->name, z->namelen);
1613                 state.origin_len = z->namelen;
1614         }
1615         /* parse the (toplevel) file */
1616         if(!az_parse_file(z, in, rr, sizeof(rr), &state, zfilename, 0, cfg)) {
1617                 char* n = sldns_wire2str_dname(z->name, z->namelen);
1618                 log_err("error parsing zonefile %s for %s",
1619                         zfilename, n?n:"error");
1620                 free(n);
1621                 fclose(in);
1622                 return 0;
1623         }
1624         fclose(in);
1625
1626         if(z->rpz)
1627                 rpz_finish_config(z->rpz);
1628         return 1;
1629 }
1630
1631 /** write buffer to file and check return codes */
1632 static int
1633 write_out(FILE* out, const char* str, size_t len)
1634 {
1635         size_t r;
1636         if(len == 0)
1637                 return 1;
1638         r = fwrite(str, 1, len, out);
1639         if(r == 0) {
1640                 log_err("write failed: %s", strerror(errno));
1641                 return 0;
1642         } else if(r < len) {
1643                 log_err("write failed: too short (disk full?)");
1644                 return 0;
1645         }
1646         return 1;
1647 }
1648
1649 /** convert auth rr to string */
1650 static int
1651 auth_rr_to_string(uint8_t* nm, size_t nmlen, uint16_t tp, uint16_t cl,
1652         struct packed_rrset_data* data, size_t i, char* s, size_t buflen)
1653 {
1654         int w = 0;
1655         size_t slen = buflen, datlen;
1656         uint8_t* dat;
1657         if(i >= data->count) tp = LDNS_RR_TYPE_RRSIG;
1658         dat = nm;
1659         datlen = nmlen;
1660         w += sldns_wire2str_dname_scan(&dat, &datlen, &s, &slen, NULL, 0, NULL);
1661         w += sldns_str_print(&s, &slen, "\t");
1662         w += sldns_str_print(&s, &slen, "%lu\t", (unsigned long)data->rr_ttl[i]);
1663         w += sldns_wire2str_class_print(&s, &slen, cl);
1664         w += sldns_str_print(&s, &slen, "\t");
1665         w += sldns_wire2str_type_print(&s, &slen, tp);
1666         w += sldns_str_print(&s, &slen, "\t");
1667         datlen = data->rr_len[i]-2;
1668         dat = data->rr_data[i]+2;
1669         w += sldns_wire2str_rdata_scan(&dat, &datlen, &s, &slen, tp, NULL, 0, NULL);
1670
1671         if(tp == LDNS_RR_TYPE_DNSKEY) {
1672                 w += sldns_str_print(&s, &slen, " ;{id = %u}",
1673                         sldns_calc_keytag_raw(data->rr_data[i]+2,
1674                                 data->rr_len[i]-2));
1675         }
1676         w += sldns_str_print(&s, &slen, "\n");
1677
1678         if(w >= (int)buflen) {
1679                 log_nametypeclass(NO_VERBOSE, "RR too long to print", nm, tp, cl);
1680                 return 0;
1681         }
1682         return 1;
1683 }
1684
1685 /** write rrset to file */
1686 static int
1687 auth_zone_write_rrset(struct auth_zone* z, struct auth_data* node,
1688         struct auth_rrset* r, FILE* out)
1689 {
1690         size_t i, count = r->data->count + r->data->rrsig_count;
1691         char buf[LDNS_RR_BUF_SIZE];
1692         for(i=0; i<count; i++) {
1693                 if(!auth_rr_to_string(node->name, node->namelen, r->type,
1694                         z->dclass, r->data, i, buf, sizeof(buf))) {
1695                         verbose(VERB_ALGO, "failed to rr2str rr %d", (int)i);
1696                         continue;
1697                 }
1698                 if(!write_out(out, buf, strlen(buf)))
1699                         return 0;
1700         }
1701         return 1;
1702 }
1703
1704 /** write domain to file */
1705 static int
1706 auth_zone_write_domain(struct auth_zone* z, struct auth_data* n, FILE* out)
1707 {
1708         struct auth_rrset* r;
1709         /* if this is zone apex, write SOA first */
1710         if(z->namelen == n->namelen) {
1711                 struct auth_rrset* soa = az_domain_rrset(n, LDNS_RR_TYPE_SOA);
1712                 if(soa) {
1713                         if(!auth_zone_write_rrset(z, n, soa, out))
1714                                 return 0;
1715                 }
1716         }
1717         /* write all the RRsets for this domain */
1718         for(r = n->rrsets; r; r = r->next) {
1719                 if(z->namelen == n->namelen &&
1720                         r->type == LDNS_RR_TYPE_SOA)
1721                         continue; /* skip SOA here */
1722                 if(!auth_zone_write_rrset(z, n, r, out))
1723                         return 0;
1724         }
1725         return 1;
1726 }
1727
1728 int auth_zone_write_file(struct auth_zone* z, const char* fname)
1729 {
1730         FILE* out;
1731         struct auth_data* n;
1732         out = fopen(fname, "w");
1733         if(!out) {
1734                 log_err("could not open %s: %s", fname, strerror(errno));
1735                 return 0;
1736         }
1737         RBTREE_FOR(n, struct auth_data*, &z->data) {
1738                 if(!auth_zone_write_domain(z, n, out)) {
1739                         log_err("could not write domain to %s", fname);
1740                         fclose(out);
1741                         return 0;
1742                 }
1743         }
1744         fclose(out);
1745         return 1;
1746 }
1747
1748 /** offline verify for zonemd, while reading a zone file to immediately
1749  * spot bad hashes in zonefile as they are read.
1750  * Creates temp buffers, but uses anchors and validation environment
1751  * from the module_env. */
1752 static void
1753 zonemd_offline_verify(struct auth_zone* z, struct module_env* env_for_val,
1754         struct module_stack* mods)
1755 {
1756         struct module_env env;
1757         time_t now = 0;
1758         if(!z->zonemd_check)
1759                 return;
1760         env = *env_for_val;
1761         env.scratch_buffer = sldns_buffer_new(env.cfg->msg_buffer_size);
1762         if(!env.scratch_buffer) {
1763                 log_err("out of memory");
1764                 goto clean_exit;
1765         }
1766         env.scratch = regional_create();
1767         if(!env.now) {
1768                 env.now = &now;
1769                 now = time(NULL);
1770         }
1771         if(!env.scratch) {
1772                 log_err("out of memory");
1773                 goto clean_exit;
1774         }
1775         auth_zone_verify_zonemd(z, &env, mods, NULL, 1, 0);
1776
1777 clean_exit:
1778         /* clean up and exit */
1779         sldns_buffer_free(env.scratch_buffer);
1780         regional_destroy(env.scratch);
1781 }
1782
1783 /** read all auth zones from file (if they have) */
1784 static int
1785 auth_zones_read_zones(struct auth_zones* az, struct config_file* cfg,
1786         struct module_env* env, struct module_stack* mods)
1787 {
1788         struct auth_zone* z;
1789         lock_rw_wrlock(&az->lock);
1790         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
1791                 lock_rw_wrlock(&z->lock);
1792                 if(!auth_zone_read_zonefile(z, cfg)) {
1793                         lock_rw_unlock(&z->lock);
1794                         lock_rw_unlock(&az->lock);
1795                         return 0;
1796                 }
1797                 if(z->zonefile && z->zonefile[0]!=0 && env)
1798                         zonemd_offline_verify(z, env, mods);
1799                 lock_rw_unlock(&z->lock);
1800         }
1801         lock_rw_unlock(&az->lock);
1802         return 1;
1803 }
1804
1805 /** fetch the content of a ZONEMD RR from the rdata */
1806 static int zonemd_fetch_parameters(struct auth_rrset* zonemd_rrset, size_t i,
1807         uint32_t* serial, int* scheme, int* hashalgo, uint8_t** hash,
1808         size_t* hashlen)
1809 {
1810         size_t rr_len;
1811         uint8_t* rdata;
1812         if(i >= zonemd_rrset->data->count)
1813                 return 0;
1814         rr_len = zonemd_rrset->data->rr_len[i];
1815         if(rr_len < 2+4+1+1)
1816                 return 0; /* too short, for rdlen+serial+scheme+algo */
1817         rdata = zonemd_rrset->data->rr_data[i];
1818         *serial = sldns_read_uint32(rdata+2);
1819         *scheme = rdata[6];
1820         *hashalgo = rdata[7];
1821         *hashlen = rr_len - 8;
1822         if(*hashlen == 0)
1823                 *hash = NULL;
1824         else    *hash = rdata+8;
1825         return 1;
1826 }
1827
1828 /**
1829  * See if the ZONEMD scheme, hash occurs more than once.
1830  * @param zonemd_rrset: the zonemd rrset to check with the RRs in it.
1831  * @param index: index of the original, this is allowed to have that
1832  *      scheme and hashalgo, but other RRs should not have it.
1833  * @param scheme: the scheme to check for.
1834  * @param hashalgo: the hash algorithm to check for.
1835  * @return true if it occurs more than once.
1836  */
1837 static int zonemd_is_duplicate_scheme_hash(struct auth_rrset* zonemd_rrset,
1838         size_t index, int scheme, int hashalgo)
1839 {
1840         size_t j;
1841         for(j=0; j<zonemd_rrset->data->count; j++) {
1842                 uint32_t serial2 = 0;
1843                 int scheme2 = 0, hashalgo2 = 0;
1844                 uint8_t* hash2 = NULL;
1845                 size_t hashlen2 = 0;
1846                 if(index == j) {
1847                         /* this is the original */
1848                         continue;
1849                 }
1850                 if(!zonemd_fetch_parameters(zonemd_rrset, j, &serial2,
1851                         &scheme2, &hashalgo2, &hash2, &hashlen2)) {
1852                         /* malformed, skip it */
1853                         continue;
1854                 }
1855                 if(scheme == scheme2 && hashalgo == hashalgo2) {
1856                         /* duplicate scheme, hash */
1857                         verbose(VERB_ALGO, "zonemd duplicate for scheme %d "
1858                                 "and hash %d", scheme, hashalgo);
1859                         return 1;
1860                 }
1861         }
1862         return 0;
1863 }
1864
1865 /**
1866  * Check ZONEMDs if present for the auth zone.  Depending on config
1867  * it can warn or fail on that.  Checks the hash of the ZONEMD.
1868  * @param z: auth zone to check for.
1869  *      caller must hold lock on zone.
1870  * @param env: module env for temp buffers.
1871  * @param reason: returned on failure.
1872  * @return false on failure, true if hash checks out.
1873  */
1874 static int auth_zone_zonemd_check_hash(struct auth_zone* z,
1875         struct module_env* env, char** reason)
1876 {
1877         /* loop over ZONEMDs and see which one is valid. if not print
1878          * failure (depending on config) */
1879         struct auth_data* apex;
1880         struct auth_rrset* zonemd_rrset;
1881         size_t i;
1882         struct regional* region = NULL;
1883         struct sldns_buffer* buf = NULL;
1884         uint32_t soa_serial = 0;
1885         region = env->scratch;
1886         regional_free_all(region);
1887         buf = env->scratch_buffer;
1888         if(!auth_zone_get_serial(z, &soa_serial)) {
1889                 *reason = "zone has no SOA serial";
1890                 return 0;
1891         }
1892
1893         apex = az_find_name(z, z->name, z->namelen);
1894         if(!apex) {
1895                 *reason = "zone has no apex";
1896                 return 0;
1897         }
1898         zonemd_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_ZONEMD);
1899         if(!zonemd_rrset || zonemd_rrset->data->count==0) {
1900                 *reason = "zone has no ZONEMD";
1901                 return 0; /* no RRset or no RRs in rrset */
1902         }
1903
1904         /* we have a ZONEMD, check if it is correct */
1905         for(i=0; i<zonemd_rrset->data->count; i++) {
1906                 uint32_t serial = 0;
1907                 int scheme = 0, hashalgo = 0;
1908                 uint8_t* hash = NULL;
1909                 size_t hashlen = 0;
1910                 if(!zonemd_fetch_parameters(zonemd_rrset, i, &serial, &scheme,
1911                         &hashalgo, &hash, &hashlen)) {
1912                         /* malformed RR */
1913                         *reason = "ZONEMD rdata malformed";
1914                         continue;
1915                 }
1916                 /* check for duplicates */
1917                 if(zonemd_is_duplicate_scheme_hash(zonemd_rrset, i, scheme,
1918                         hashalgo)) {
1919                         /* duplicate hash of the same scheme,hash
1920                          * is not allowed. */
1921                         *reason = "ZONEMD RRSet contains more than one RR "
1922                                 "with the same scheme and hash algorithm";
1923                         continue;
1924                 }
1925                 regional_free_all(region);
1926                 if(serial != soa_serial) {
1927                         *reason = "ZONEMD serial is wrong";
1928                         continue;
1929                 }
1930                 if(auth_zone_generate_zonemd_check(z, scheme, hashalgo,
1931                         hash, hashlen, region, buf, reason)) {
1932                         /* success */
1933                         if(verbosity >= VERB_ALGO) {
1934                                 char zstr[255+1];
1935                                 dname_str(z->name, zstr);
1936                                 verbose(VERB_ALGO, "auth-zone %s ZONEMD hash is correct", zstr);
1937                         }
1938                         return 1;
1939                 }
1940                 /* try next one */
1941         }
1942         /* fail, we may have reason */
1943         if(!*reason)
1944                 *reason = "no ZONEMD records found";
1945         if(verbosity >= VERB_ALGO) {
1946                 char zstr[255+1];
1947                 dname_str(z->name, zstr);
1948                 verbose(VERB_ALGO, "auth-zone %s ZONEMD failed: %s", zstr, *reason);
1949         }
1950         return 0;
1951 }
1952
1953 /** find the apex SOA RRset, if it exists */
1954 struct auth_rrset* auth_zone_get_soa_rrset(struct auth_zone* z)
1955 {
1956         struct auth_data* apex;
1957         struct auth_rrset* soa;
1958         apex = az_find_name(z, z->name, z->namelen);
1959         if(!apex) return NULL;
1960         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1961         return soa;
1962 }
1963
1964 /** find serial number of zone or false if none */
1965 int
1966 auth_zone_get_serial(struct auth_zone* z, uint32_t* serial)
1967 {
1968         struct auth_data* apex;
1969         struct auth_rrset* soa;
1970         struct packed_rrset_data* d;
1971         apex = az_find_name(z, z->name, z->namelen);
1972         if(!apex) return 0;
1973         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1974         if(!soa || soa->data->count==0)
1975                 return 0; /* no RRset or no RRs in rrset */
1976         if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1977         d = soa->data;
1978         *serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
1979         return 1;
1980 }
1981
1982 /** Find auth_zone SOA and populate the values in xfr(soa values). */
1983 int
1984 xfr_find_soa(struct auth_zone* z, struct auth_xfer* xfr)
1985 {
1986         struct auth_data* apex;
1987         struct auth_rrset* soa;
1988         struct packed_rrset_data* d;
1989         apex = az_find_name(z, z->name, z->namelen);
1990         if(!apex) return 0;
1991         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
1992         if(!soa || soa->data->count==0)
1993                 return 0; /* no RRset or no RRs in rrset */
1994         if(soa->data->rr_len[0] < 2+4*5) return 0; /* SOA too short */
1995         /* SOA record ends with serial, refresh, retry, expiry, minimum,
1996          * as 4 byte fields */
1997         d = soa->data;
1998         xfr->have_zone = 1;
1999         xfr->serial = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-20));
2000         xfr->refresh = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-16));
2001         xfr->retry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-12));
2002         xfr->expiry = sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-8));
2003         /* soa minimum at d->rr_len[0]-4 */
2004         return 1;
2005 }
2006
2007 /**
2008  * Setup auth_xfer zone
2009  * This populates the have_zone, soa values, and so on times.
2010  * Doesn't do network traffic yet, can set option flags.
2011  * @param z: locked by caller, and modified for setup
2012  * @param x: locked by caller, and modified.
2013  * @return false on failure.
2014  */
2015 static int
2016 auth_xfer_setup(struct auth_zone* z, struct auth_xfer* x)
2017 {
2018         /* for a zone without zone transfers, x==NULL, so skip them,
2019          * i.e. the zone config is fixed with no masters or urls */
2020         if(!z || !x) return 1;
2021         if(!xfr_find_soa(z, x)) {
2022                 return 1;
2023         }
2024         /* nothing for probe, nextprobe and transfer tasks */
2025         return 1;
2026 }
2027
2028 /**
2029  * Setup all zones
2030  * @param az: auth zones structure
2031  * @return false on failure.
2032  */
2033 static int
2034 auth_zones_setup_zones(struct auth_zones* az)
2035 {
2036         struct auth_zone* z;
2037         struct auth_xfer* x;
2038         lock_rw_wrlock(&az->lock);
2039         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2040                 lock_rw_wrlock(&z->lock);
2041                 x = auth_xfer_find(az, z->name, z->namelen, z->dclass);
2042                 if(x) {
2043                         lock_basic_lock(&x->lock);
2044                 }
2045                 if(!auth_xfer_setup(z, x)) {
2046                         if(x) {
2047                                 lock_basic_unlock(&x->lock);
2048                         }
2049                         lock_rw_unlock(&z->lock);
2050                         lock_rw_unlock(&az->lock);
2051                         return 0;
2052                 }
2053                 if(x) {
2054                         lock_basic_unlock(&x->lock);
2055                 }
2056                 lock_rw_unlock(&z->lock);
2057         }
2058         lock_rw_unlock(&az->lock);
2059         return 1;
2060 }
2061
2062 /** set config items and create zones */
2063 static int
2064 auth_zones_cfg(struct auth_zones* az, struct config_auth* c)
2065 {
2066         struct auth_zone* z;
2067         struct auth_xfer* x = NULL;
2068
2069         /* create zone */
2070         if(c->isrpz) {
2071                 /* if the rpz lock is needed, grab it before the other
2072                  * locks to avoid a lock dependency cycle */
2073                 lock_rw_wrlock(&az->rpz_lock);
2074         }
2075         lock_rw_wrlock(&az->lock);
2076         if(!(z=auth_zones_find_or_add_zone(az, c->name))) {
2077                 lock_rw_unlock(&az->lock);
2078                 if(c->isrpz) {
2079                         lock_rw_unlock(&az->rpz_lock);
2080                 }
2081                 return 0;
2082         }
2083         if(c->masters || c->urls) {
2084                 if(!(x=auth_zones_find_or_add_xfer(az, z))) {
2085                         lock_rw_unlock(&az->lock);
2086                         lock_rw_unlock(&z->lock);
2087                         if(c->isrpz) {
2088                                 lock_rw_unlock(&az->rpz_lock);
2089                         }
2090                         return 0;
2091                 }
2092         }
2093         if(c->for_downstream)
2094                 az->have_downstream = 1;
2095         lock_rw_unlock(&az->lock);
2096
2097         /* set options */
2098         z->zone_deleted = 0;
2099         if(!auth_zone_set_zonefile(z, c->zonefile)) {
2100                 if(x) {
2101                         lock_basic_unlock(&x->lock);
2102                 }
2103                 lock_rw_unlock(&z->lock);
2104                 if(c->isrpz) {
2105                         lock_rw_unlock(&az->rpz_lock);
2106                 }
2107                 return 0;
2108         }
2109         z->for_downstream = c->for_downstream;
2110         z->for_upstream = c->for_upstream;
2111         z->fallback_enabled = c->fallback_enabled;
2112         z->zonemd_check = c->zonemd_check;
2113         z->zonemd_reject_absence = c->zonemd_reject_absence;
2114         if(c->isrpz && !z->rpz){
2115                 if(!(z->rpz = rpz_create(c))){
2116                         fatal_exit("Could not setup RPZ zones");
2117                         return 0;
2118                 }
2119                 lock_protect(&z->lock, &z->rpz->local_zones, sizeof(*z->rpz));
2120                 /* the az->rpz_lock is locked above */
2121                 z->rpz_az_next = az->rpz_first;
2122                 if(az->rpz_first)
2123                         az->rpz_first->rpz_az_prev = z;
2124                 az->rpz_first = z;
2125         }
2126         if(c->isrpz) {
2127                 lock_rw_unlock(&az->rpz_lock);
2128         }
2129
2130         /* xfer zone */
2131         if(x) {
2132                 z->zone_is_slave = 1;
2133                 /* set options on xfer zone */
2134                 if(!xfer_set_masters(&x->task_probe->masters, c, 0)) {
2135                         lock_basic_unlock(&x->lock);
2136                         lock_rw_unlock(&z->lock);
2137                         return 0;
2138                 }
2139                 if(!xfer_set_masters(&x->task_transfer->masters, c, 1)) {
2140                         lock_basic_unlock(&x->lock);
2141                         lock_rw_unlock(&z->lock);
2142                         return 0;
2143                 }
2144                 lock_basic_unlock(&x->lock);
2145         }
2146
2147         lock_rw_unlock(&z->lock);
2148         return 1;
2149 }
2150
2151 /** set all auth zones deleted, then in auth_zones_cfg, it marks them
2152  * as nondeleted (if they are still in the config), and then later
2153  * we can find deleted zones */
2154 static void
2155 az_setall_deleted(struct auth_zones* az)
2156 {
2157         struct auth_zone* z;
2158         lock_rw_wrlock(&az->lock);
2159         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2160                 lock_rw_wrlock(&z->lock);
2161                 z->zone_deleted = 1;
2162                 lock_rw_unlock(&z->lock);
2163         }
2164         lock_rw_unlock(&az->lock);
2165 }
2166
2167 /** find zones that are marked deleted and delete them.
2168  * This is called from apply_cfg, and there are no threads and no
2169  * workers, so the xfr can just be deleted. */
2170 static void
2171 az_delete_deleted_zones(struct auth_zones* az)
2172 {
2173         struct auth_zone* z;
2174         struct auth_zone* delete_list = NULL, *next;
2175         struct auth_xfer* xfr;
2176         lock_rw_wrlock(&az->lock);
2177         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
2178                 lock_rw_wrlock(&z->lock);
2179                 if(z->zone_deleted) {
2180                         /* we cannot alter the rbtree right now, but
2181                          * we can put it on a linked list and then
2182                          * delete it */
2183                         z->delete_next = delete_list;
2184                         delete_list = z;
2185                 }
2186                 lock_rw_unlock(&z->lock);
2187         }
2188         /* now we are out of the tree loop and we can loop and delete
2189          * the zones */
2190         z = delete_list;
2191         while(z) {
2192                 next = z->delete_next;
2193                 xfr = auth_xfer_find(az, z->name, z->namelen, z->dclass);
2194                 if(xfr) {
2195                         (void)rbtree_delete(&az->xtree, &xfr->node);
2196                         auth_xfer_delete(xfr);
2197                 }
2198                 (void)rbtree_delete(&az->ztree, &z->node);
2199                 auth_zone_delete(z, az);
2200                 z = next;
2201         }
2202         lock_rw_unlock(&az->lock);
2203 }
2204
2205 int auth_zones_apply_cfg(struct auth_zones* az, struct config_file* cfg,
2206         int setup, int* is_rpz, struct module_env* env,
2207         struct module_stack* mods)
2208 {
2209         struct config_auth* p;
2210         az_setall_deleted(az);
2211         for(p = cfg->auths; p; p = p->next) {
2212                 if(!p->name || p->name[0] == 0) {
2213                         log_warn("auth-zone without a name, skipped");
2214                         continue;
2215                 }
2216                 *is_rpz = (*is_rpz || p->isrpz);
2217                 if(!auth_zones_cfg(az, p)) {
2218                         log_err("cannot config auth zone %s", p->name);
2219                         return 0;
2220                 }
2221         }
2222         az_delete_deleted_zones(az);
2223         if(!auth_zones_read_zones(az, cfg, env, mods))
2224                 return 0;
2225         if(setup) {
2226                 if(!auth_zones_setup_zones(az))
2227                         return 0;
2228         }
2229         return 1;
2230 }
2231
2232 /** delete chunks
2233  * @param at: transfer structure with chunks list.  The chunks and their
2234  *      data are freed.
2235  */
2236 static void
2237 auth_chunks_delete(struct auth_transfer* at)
2238 {
2239         if(at->chunks_first) {
2240                 struct auth_chunk* c, *cn;
2241                 c = at->chunks_first;
2242                 while(c) {
2243                         cn = c->next;
2244                         free(c->data);
2245                         free(c);
2246                         c = cn;
2247                 }
2248         }
2249         at->chunks_first = NULL;
2250         at->chunks_last = NULL;
2251 }
2252
2253 /** free master addr list */
2254 static void
2255 auth_free_master_addrs(struct auth_addr* list)
2256 {
2257         struct auth_addr *n;
2258         while(list) {
2259                 n = list->next;
2260                 free(list);
2261                 list = n;
2262         }
2263 }
2264
2265 /** free the masters list */
2266 static void
2267 auth_free_masters(struct auth_master* list)
2268 {
2269         struct auth_master* n;
2270         while(list) {
2271                 n = list->next;
2272                 auth_free_master_addrs(list->list);
2273                 free(list->host);
2274                 free(list->file);
2275                 free(list);
2276                 list = n;
2277         }
2278 }
2279
2280 /** delete auth xfer structure
2281  * @param xfr: delete this xfer and its tasks.
2282  */
2283 static void
2284 auth_xfer_delete(struct auth_xfer* xfr)
2285 {
2286         if(!xfr) return;
2287         lock_basic_destroy(&xfr->lock);
2288         free(xfr->name);
2289         if(xfr->task_nextprobe) {
2290                 comm_timer_delete(xfr->task_nextprobe->timer);
2291                 free(xfr->task_nextprobe);
2292         }
2293         if(xfr->task_probe) {
2294                 auth_free_masters(xfr->task_probe->masters);
2295                 comm_point_delete(xfr->task_probe->cp);
2296                 comm_timer_delete(xfr->task_probe->timer);
2297                 free(xfr->task_probe);
2298         }
2299         if(xfr->task_transfer) {
2300                 auth_free_masters(xfr->task_transfer->masters);
2301                 comm_point_delete(xfr->task_transfer->cp);
2302                 comm_timer_delete(xfr->task_transfer->timer);
2303                 if(xfr->task_transfer->chunks_first) {
2304                         auth_chunks_delete(xfr->task_transfer);
2305                 }
2306                 free(xfr->task_transfer);
2307         }
2308         auth_free_masters(xfr->allow_notify_list);
2309         free(xfr);
2310 }
2311
2312 /** helper traverse to delete zones */
2313 static void
2314 auth_zone_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2315 {
2316         struct auth_zone* z = (struct auth_zone*)n->key;
2317         auth_zone_delete(z, NULL);
2318 }
2319
2320 /** helper traverse to delete xfer zones */
2321 static void
2322 auth_xfer_del(rbnode_type* n, void* ATTR_UNUSED(arg))
2323 {
2324         struct auth_xfer* z = (struct auth_xfer*)n->key;
2325         auth_xfer_delete(z);
2326 }
2327
2328 void auth_zones_delete(struct auth_zones* az)
2329 {
2330         if(!az) return;
2331         lock_rw_destroy(&az->lock);
2332         lock_rw_destroy(&az->rpz_lock);
2333         traverse_postorder(&az->ztree, auth_zone_del, NULL);
2334         traverse_postorder(&az->xtree, auth_xfer_del, NULL);
2335         free(az);
2336 }
2337
2338 /** true if domain has only nsec3 */
2339 static int
2340 domain_has_only_nsec3(struct auth_data* n)
2341 {
2342         struct auth_rrset* rrset = n->rrsets;
2343         int nsec3_seen = 0;
2344         while(rrset) {
2345                 if(rrset->type == LDNS_RR_TYPE_NSEC3) {
2346                         nsec3_seen = 1;
2347                 } else if(rrset->type != LDNS_RR_TYPE_RRSIG) {
2348                         return 0;
2349                 }
2350                 rrset = rrset->next;
2351         }
2352         return nsec3_seen;
2353 }
2354
2355 /** see if the domain has a wildcard child '*.domain' */
2356 static struct auth_data*
2357 az_find_wildcard_domain(struct auth_zone* z, uint8_t* nm, size_t nmlen)
2358 {
2359         uint8_t wc[LDNS_MAX_DOMAINLEN];
2360         if(nmlen+2 > sizeof(wc))
2361                 return NULL; /* result would be too long */
2362         wc[0] = 1; /* length of wildcard label */
2363         wc[1] = (uint8_t)'*'; /* wildcard label */
2364         memmove(wc+2, nm, nmlen);
2365         return az_find_name(z, wc, nmlen+2);
2366 }
2367
2368 /** find wildcard between qname and cename */
2369 static struct auth_data*
2370 az_find_wildcard(struct auth_zone* z, struct query_info* qinfo,
2371         struct auth_data* ce)
2372 {
2373         uint8_t* nm = qinfo->qname;
2374         size_t nmlen = qinfo->qname_len;
2375         struct auth_data* node;
2376         if(!dname_subdomain_c(nm, z->name))
2377                 return NULL; /* out of zone */
2378         while((node=az_find_wildcard_domain(z, nm, nmlen))==NULL) {
2379                 /* see if we can go up to find the wildcard */
2380                 if(nmlen == z->namelen)
2381                         return NULL; /* top of zone reached */
2382                 if(ce && nmlen == ce->namelen)
2383                         return NULL; /* ce reached */
2384                 if(dname_is_root(nm))
2385                         return NULL; /* cannot go up */
2386                 dname_remove_label(&nm, &nmlen);
2387         }
2388         return node;
2389 }
2390
2391 /** domain is not exact, find first candidate ce (name that matches
2392  * a part of qname) in tree */
2393 static struct auth_data*
2394 az_find_candidate_ce(struct auth_zone* z, struct query_info* qinfo,
2395         struct auth_data* n)
2396 {
2397         uint8_t* nm;
2398         size_t nmlen;
2399         if(n) {
2400                 nm = dname_get_shared_topdomain(qinfo->qname, n->name);
2401         } else {
2402                 nm = qinfo->qname;
2403         }
2404         dname_count_size_labels(nm, &nmlen);
2405         n = az_find_name(z, nm, nmlen);
2406         /* delete labels and go up on name */
2407         while(!n) {
2408                 if(dname_is_root(nm))
2409                         return NULL; /* cannot go up */
2410                 dname_remove_label(&nm, &nmlen);
2411                 n = az_find_name(z, nm, nmlen);
2412         }
2413         return n;
2414 }
2415
2416 /** go up the auth tree to next existing name. */
2417 static struct auth_data*
2418 az_domain_go_up(struct auth_zone* z, struct auth_data* n)
2419 {
2420         uint8_t* nm = n->name;
2421         size_t nmlen = n->namelen;
2422         while(!dname_is_root(nm)) {
2423                 dname_remove_label(&nm, &nmlen);
2424                 if((n=az_find_name(z, nm, nmlen)) != NULL)
2425                         return n;
2426         }
2427         return NULL;
2428 }
2429
2430 /** Find the closest encloser, an name that exists and is above the
2431  * qname.
2432  * return true if the node (param node) is existing, nonobscured and
2433  *      can be used to generate answers from.  It is then also node_exact.
2434  * returns false if the node is not good enough (or it wasn't node_exact)
2435  *      in this case the ce can be filled.
2436  *      if ce is NULL, no ce exists, and likely the zone is completely empty,
2437  *      not even with a zone apex.
2438  *      if ce is nonNULL it is the closest enclosing upper name (that exists
2439  *      itself for answer purposes).  That name may have DNAME, NS or wildcard
2440  *      rrset is the closest DNAME or NS rrset that was found.
2441  */
2442 static int
2443 az_find_ce(struct auth_zone* z, struct query_info* qinfo,
2444         struct auth_data* node, int node_exact, struct auth_data** ce,
2445         struct auth_rrset** rrset)
2446 {
2447         struct auth_data* n = node;
2448         *ce = NULL;
2449         *rrset = NULL;
2450         if(!node_exact) {
2451                 /* if not exact, lookup closest exact match */
2452                 n = az_find_candidate_ce(z, qinfo, n);
2453         } else {
2454                 /* if exact, the node itself is the first candidate ce */
2455                 *ce = n;
2456         }
2457
2458         /* no direct answer from nsec3-only domains */
2459         if(n && domain_has_only_nsec3(n)) {
2460                 node_exact = 0;
2461                 *ce = NULL;
2462         }
2463
2464         /* with exact matches, walk up the labels until we find the
2465          * delegation, or DNAME or zone end */
2466         while(n) {
2467                 /* see if the current candidate has issues */
2468                 /* not zone apex and has type NS */
2469                 if(n->namelen != z->namelen &&
2470                         (*rrset=az_domain_rrset(n, LDNS_RR_TYPE_NS)) &&
2471                         /* delegate here, but DS at exact the dp has notype */
2472                         (qinfo->qtype != LDNS_RR_TYPE_DS ||
2473                         n->namelen != qinfo->qname_len)) {
2474                         /* referral */
2475                         /* this is ce and the lowernode is nonexisting */
2476                         *ce = n;
2477                         return 0;
2478                 }
2479                 /* not equal to qname and has type DNAME */
2480                 if(n->namelen != qinfo->qname_len &&
2481                         (*rrset=az_domain_rrset(n, LDNS_RR_TYPE_DNAME))) {
2482                         /* this is ce and the lowernode is nonexisting */
2483                         *ce = n;
2484                         return 0;
2485                 }
2486
2487                 if(*ce == NULL && !domain_has_only_nsec3(n)) {
2488                         /* if not found yet, this exact name must be
2489                          * our lowest match (but not nsec3onlydomain) */
2490                         *ce = n;
2491                 }
2492
2493                 /* walk up the tree by removing labels from name and lookup */
2494                 n = az_domain_go_up(z, n);
2495         }
2496         /* found no problems, if it was an exact node, it is fine to use */
2497         return node_exact;
2498 }
2499
2500 /** add additional A/AAAA from domain names in rrset rdata (+offset)
2501  * offset is number of bytes in rdata where the dname is located. */
2502 static int
2503 az_add_additionals_from(struct auth_zone* z, struct regional* region,
2504         struct dns_msg* msg, struct auth_rrset* rrset, size_t offset)
2505 {
2506         struct packed_rrset_data* d = rrset->data;
2507         size_t i;
2508         if(!d) return 0;
2509         for(i=0; i<d->count; i++) {
2510                 size_t dlen;
2511                 struct auth_data* domain;
2512                 struct auth_rrset* ref;
2513                 if(d->rr_len[i] < 2+offset)
2514                         continue; /* too short */
2515                 if(!(dlen = dname_valid(d->rr_data[i]+2+offset,
2516                         d->rr_len[i]-2-offset)))
2517                         continue; /* malformed */
2518                 domain = az_find_name(z, d->rr_data[i]+2+offset, dlen);
2519                 if(!domain)
2520                         continue;
2521                 if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_A)) != NULL) {
2522                         if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2523                                 return 0;
2524                 }
2525                 if((ref=az_domain_rrset(domain, LDNS_RR_TYPE_AAAA)) != NULL) {
2526                         if(!msg_add_rrset_ar(z, region, msg, domain, ref))
2527                                 return 0;
2528                 }
2529         }
2530         return 1;
2531 }
2532
2533 /** add negative SOA record (with negative TTL) */
2534 static int
2535 az_add_negative_soa(struct auth_zone* z, struct regional* region,
2536         struct dns_msg* msg)
2537 {
2538         time_t minimum;
2539         size_t i;
2540         struct packed_rrset_data* d;
2541         struct auth_rrset* soa;
2542         struct auth_data* apex = az_find_name(z, z->name, z->namelen);
2543         if(!apex) return 0;
2544         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
2545         if(!soa) return 0;
2546         /* must be first to put in message; we want to fix the TTL with
2547          * one RRset here, otherwise we'd need to loop over the RRs to get
2548          * the resulting lower TTL */
2549         log_assert(msg->rep->rrset_count == 0);
2550         if(!msg_add_rrset_ns(z, region, msg, apex, soa)) return 0;
2551         /* fixup TTL */
2552         d = (struct packed_rrset_data*)msg->rep->rrsets[msg->rep->rrset_count-1]->entry.data;
2553         /* last 4 bytes are minimum ttl in network format */
2554         if(d->count == 0) return 0;
2555         if(d->rr_len[0] < 2+4) return 0;
2556         minimum = (time_t)sldns_read_uint32(d->rr_data[0]+(d->rr_len[0]-4));
2557         minimum = d->ttl<minimum?d->ttl:minimum;
2558         d->ttl = minimum;
2559         for(i=0; i < d->count + d->rrsig_count; i++)
2560                 d->rr_ttl[i] = minimum;
2561         msg->rep->ttl = get_rrset_ttl(msg->rep->rrsets[0]);
2562         msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
2563         msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
2564         return 1;
2565 }
2566
2567 /** See if the query goes to empty nonterminal (that has no auth_data,
2568  * but there are nodes underneath.  We already checked that there are
2569  * not NS, or DNAME above, so that we only need to check if some node
2570  * exists below (with nonempty rr list), return true if emptynonterminal */
2571 static int
2572 az_empty_nonterminal(struct auth_zone* z, struct query_info* qinfo,
2573         struct auth_data* node)
2574 {
2575         struct auth_data* next;
2576         if(!node) {
2577                 /* no smaller was found, use first (smallest) node as the
2578                  * next one */
2579                 next = (struct auth_data*)rbtree_first(&z->data);
2580         } else {
2581                 next = (struct auth_data*)rbtree_next(&node->node);
2582         }
2583         while(next && (rbnode_type*)next != RBTREE_NULL && next->rrsets == NULL) {
2584                 /* the next name has empty rrsets, is an empty nonterminal
2585                  * itself, see if there exists something below it */
2586                 next = (struct auth_data*)rbtree_next(&node->node);
2587         }
2588         if((rbnode_type*)next == RBTREE_NULL || !next) {
2589                 /* there is no next node, so something below it cannot
2590                  * exist */
2591                 return 0;
2592         }
2593         /* a next node exists, if there was something below the query,
2594          * this node has to be it.  See if it is below the query name */
2595         if(dname_strict_subdomain_c(next->name, qinfo->qname))
2596                 return 1;
2597         return 0;
2598 }
2599
2600 /** create synth cname target name in buffer, or fail if too long */
2601 static size_t
2602 synth_cname_buf(uint8_t* qname, size_t qname_len, size_t dname_len,
2603         uint8_t* dtarg, size_t dtarglen, uint8_t* buf, size_t buflen)
2604 {
2605         size_t newlen = qname_len + dtarglen - dname_len;
2606         if(newlen > buflen) {
2607                 /* YXDOMAIN error */
2608                 return 0;
2609         }
2610         /* new name is concatenation of qname front (without DNAME owner)
2611          * and DNAME target name */
2612         memcpy(buf, qname, qname_len-dname_len);
2613         memmove(buf+(qname_len-dname_len), dtarg, dtarglen);
2614         return newlen;
2615 }
2616
2617 /** create synthetic CNAME rrset for in a DNAME answer in region,
2618  * false on alloc failure, cname==NULL when name too long. */
2619 static int
2620 create_synth_cname(uint8_t* qname, size_t qname_len, struct regional* region,
2621         struct auth_data* node, struct auth_rrset* dname, uint16_t dclass,
2622         struct ub_packed_rrset_key** cname)
2623 {
2624         uint8_t buf[LDNS_MAX_DOMAINLEN];
2625         uint8_t* dtarg;
2626         size_t dtarglen, newlen;
2627         struct packed_rrset_data* d;
2628
2629         /* get DNAME target name */
2630         if(dname->data->count < 1) return 0;
2631         if(dname->data->rr_len[0] < 3) return 0; /* at least rdatalen +1 */
2632         dtarg = dname->data->rr_data[0]+2;
2633         dtarglen = dname->data->rr_len[0]-2;
2634         if(sldns_read_uint16(dname->data->rr_data[0]) != dtarglen)
2635                 return 0; /* rdatalen in DNAME rdata is malformed */
2636         if(dname_valid(dtarg, dtarglen) != dtarglen)
2637                 return 0; /* DNAME RR has malformed rdata */
2638         if(qname_len == 0)
2639                 return 0; /* too short */
2640         if(qname_len <= node->namelen)
2641                 return 0; /* qname too short for dname removal */
2642
2643         /* synthesize a CNAME */
2644         newlen = synth_cname_buf(qname, qname_len, node->namelen,
2645                 dtarg, dtarglen, buf, sizeof(buf));
2646         if(newlen == 0) {
2647                 /* YXDOMAIN error */
2648                 *cname = NULL;
2649                 return 1;
2650         }
2651         *cname = (struct ub_packed_rrset_key*)regional_alloc(region,
2652                 sizeof(struct ub_packed_rrset_key));
2653         if(!*cname)
2654                 return 0; /* out of memory */
2655         memset(&(*cname)->entry, 0, sizeof((*cname)->entry));
2656         (*cname)->entry.key = (*cname);
2657         (*cname)->rk.type = htons(LDNS_RR_TYPE_CNAME);
2658         (*cname)->rk.rrset_class = htons(dclass);
2659         (*cname)->rk.flags = 0;
2660         (*cname)->rk.dname = regional_alloc_init(region, qname, qname_len);
2661         if(!(*cname)->rk.dname)
2662                 return 0; /* out of memory */
2663         (*cname)->rk.dname_len = qname_len;
2664         (*cname)->entry.hash = rrset_key_hash(&(*cname)->rk);
2665         d = (struct packed_rrset_data*)regional_alloc_zero(region,
2666                 sizeof(struct packed_rrset_data) + sizeof(size_t) +
2667                 sizeof(uint8_t*) + sizeof(time_t) + sizeof(uint16_t)
2668                 + newlen);
2669         if(!d)
2670                 return 0; /* out of memory */
2671         (*cname)->entry.data = d;
2672         d->ttl = 0; /* 0 for synthesized CNAME TTL */
2673         d->count = 1;
2674         d->rrsig_count = 0;
2675         d->trust = rrset_trust_ans_noAA;
2676         d->rr_len = (size_t*)((uint8_t*)d +
2677                 sizeof(struct packed_rrset_data));
2678         d->rr_len[0] = newlen + sizeof(uint16_t);
2679         packed_rrset_ptr_fixup(d);
2680         d->rr_ttl[0] = d->ttl;
2681         sldns_write_uint16(d->rr_data[0], newlen);
2682         memmove(d->rr_data[0] + sizeof(uint16_t), buf, newlen);
2683         return 1;
2684 }
2685
2686 /** add a synthesized CNAME to the answer section */
2687 static int
2688 add_synth_cname(struct auth_zone* z, uint8_t* qname, size_t qname_len,
2689         struct regional* region, struct dns_msg* msg, struct auth_data* dname,
2690         struct auth_rrset* rrset)
2691 {
2692         struct ub_packed_rrset_key* cname;
2693         /* synthesize a CNAME */
2694         if(!create_synth_cname(qname, qname_len, region, dname, rrset,
2695                 z->dclass, &cname)) {
2696                 /* out of memory */
2697                 return 0;
2698         }
2699         if(!cname) {
2700                 /* cname cannot be create because of YXDOMAIN */
2701                 msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
2702                 return 1;
2703         }
2704         /* add cname to message */
2705         if(!msg_grow_array(region, msg))
2706                 return 0;
2707         msg->rep->rrsets[msg->rep->rrset_count] = cname;
2708         msg->rep->rrset_count++;
2709         msg->rep->an_numrrsets++;
2710         msg_ttl(msg);
2711         return 1;
2712 }
2713
2714 /** Change a dname to a different one, for wildcard namechange */
2715 static void
2716 az_change_dnames(struct dns_msg* msg, uint8_t* oldname, uint8_t* newname,
2717         size_t newlen, int an_only)
2718 {
2719         size_t i;
2720         size_t start = 0, end = msg->rep->rrset_count;
2721         if(!an_only) start = msg->rep->an_numrrsets;
2722         if(an_only) end = msg->rep->an_numrrsets;
2723         for(i=start; i<end; i++) {
2724                 /* allocated in region so we can change the ptrs */
2725                 if(query_dname_compare(msg->rep->rrsets[i]->rk.dname, oldname)
2726                         == 0) {
2727                         msg->rep->rrsets[i]->rk.dname = newname;
2728                         msg->rep->rrsets[i]->rk.dname_len = newlen;
2729                 }
2730         }
2731 }
2732
2733 /** find NSEC record covering the query */
2734 static struct auth_rrset*
2735 az_find_nsec_cover(struct auth_zone* z, struct auth_data** node)
2736 {
2737         uint8_t* nm = (*node)->name;
2738         size_t nmlen = (*node)->namelen;
2739         struct auth_rrset* rrset;
2740         /* find the NSEC for the smallest-or-equal node */
2741         /* if node == NULL, we did not find a smaller name.  But the zone
2742          * name is the smallest name and should have an NSEC. So there is
2743          * no NSEC to return (for a properly signed zone) */
2744         /* for empty nonterminals, the auth-data node should not exist,
2745          * and thus we don't need to go rbtree_previous here to find
2746          * a domain with an NSEC record */
2747         /* but there could be glue, and if this is node, then it has no NSEC.
2748          * Go up to find nonglue (previous) NSEC-holding nodes */
2749         while((rrset=az_domain_rrset(*node, LDNS_RR_TYPE_NSEC)) == NULL) {
2750                 if(dname_is_root(nm)) return NULL;
2751                 if(nmlen == z->namelen) return NULL;
2752                 dname_remove_label(&nm, &nmlen);
2753                 /* adjust *node for the nsec rrset to find in */
2754                 *node = az_find_name(z, nm, nmlen);
2755         }
2756         return rrset;
2757 }
2758
2759 /** Find NSEC and add for wildcard denial */
2760 static int
2761 az_nsec_wildcard_denial(struct auth_zone* z, struct regional* region,
2762         struct dns_msg* msg, uint8_t* cenm, size_t cenmlen)
2763 {
2764         struct query_info qinfo;
2765         int node_exact;
2766         struct auth_data* node;
2767         struct auth_rrset* nsec;
2768         uint8_t wc[LDNS_MAX_DOMAINLEN];
2769         if(cenmlen+2 > sizeof(wc))
2770                 return 0; /* result would be too long */
2771         wc[0] = 1; /* length of wildcard label */
2772         wc[1] = (uint8_t)'*'; /* wildcard label */
2773         memmove(wc+2, cenm, cenmlen);
2774
2775         /* we have '*.ce' in wc wildcard name buffer */
2776         /* get nsec cover for that */
2777         qinfo.qname = wc;
2778         qinfo.qname_len = cenmlen+2;
2779         qinfo.qtype = 0;
2780         qinfo.qclass = 0;
2781         az_find_domain(z, &qinfo, &node_exact, &node);
2782         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
2783                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
2784         }
2785         return 1;
2786 }
2787
2788 /** Find the NSEC3PARAM rrset (if any) and if true you have the parameters */
2789 static int
2790 az_nsec3_param(struct auth_zone* z, int* algo, size_t* iter, uint8_t** salt,
2791         size_t* saltlen)
2792 {
2793         struct auth_data* apex;
2794         struct auth_rrset* param;
2795         size_t i;
2796         apex = az_find_name(z, z->name, z->namelen);
2797         if(!apex) return 0;
2798         param = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC3PARAM);
2799         if(!param || param->data->count==0)
2800                 return 0; /* no RRset or no RRs in rrset */
2801         /* find out which NSEC3PARAM RR has supported parameters */
2802         /* skip unknown flags (dynamic signer is recalculating nsec3 chain) */
2803         for(i=0; i<param->data->count; i++) {
2804                 uint8_t* rdata = param->data->rr_data[i]+2;
2805                 size_t rdatalen = param->data->rr_len[i];
2806                 if(rdatalen < 2+5)
2807                         continue; /* too short */
2808                 if(!nsec3_hash_algo_size_supported((int)(rdata[0])))
2809                         continue; /* unsupported algo */
2810                 if(rdatalen < (size_t)(2+5+(size_t)rdata[4]))
2811                         continue; /* salt missing */
2812                 if((rdata[1]&NSEC3_UNKNOWN_FLAGS)!=0)
2813                         continue; /* unknown flags */
2814                 *algo = (int)(rdata[0]);
2815                 *iter = sldns_read_uint16(rdata+2);
2816                 *saltlen = rdata[4];
2817                 if(*saltlen == 0)
2818                         *salt = NULL;
2819                 else    *salt = rdata+5;
2820                 return 1;
2821         }
2822         /* no supported params */
2823         return 0;
2824 }
2825
2826 /** Hash a name with nsec3param into buffer, it has zone name appended.
2827  * return length of hash */
2828 static size_t
2829 az_nsec3_hash(uint8_t* buf, size_t buflen, uint8_t* nm, size_t nmlen,
2830         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2831 {
2832         size_t hlen = nsec3_hash_algo_size_supported(algo);
2833         /* buffer has domain name, nsec3hash, and 256 is for max saltlen
2834          * (salt has 0-255 length) */
2835         unsigned char p[LDNS_MAX_DOMAINLEN+1+N3HASHBUFLEN+256];
2836         size_t i;
2837         if(nmlen+saltlen > sizeof(p) || hlen+saltlen > sizeof(p))
2838                 return 0;
2839         if(hlen > buflen)
2840                 return 0; /* somehow too large for destination buffer */
2841         /* hashfunc(name, salt) */
2842         memmove(p, nm, nmlen);
2843         query_dname_tolower(p);
2844         if(salt && saltlen > 0)
2845                 memmove(p+nmlen, salt, saltlen);
2846         (void)secalgo_nsec3_hash(algo, p, nmlen+saltlen, (unsigned char*)buf);
2847         for(i=0; i<iter; i++) {
2848                 /* hashfunc(hash, salt) */
2849                 memmove(p, buf, hlen);
2850                 if(salt && saltlen > 0)
2851                         memmove(p+hlen, salt, saltlen);
2852                 (void)secalgo_nsec3_hash(algo, p, hlen+saltlen,
2853                         (unsigned char*)buf);
2854         }
2855         return hlen;
2856 }
2857
2858 /** Hash name and return b32encoded hashname for lookup, zone name appended */
2859 static int
2860 az_nsec3_hashname(struct auth_zone* z, uint8_t* hashname, size_t* hashnmlen,
2861         uint8_t* nm, size_t nmlen, int algo, size_t iter, uint8_t* salt,
2862         size_t saltlen)
2863 {
2864         uint8_t hash[N3HASHBUFLEN];
2865         size_t hlen;
2866         int ret;
2867         hlen = az_nsec3_hash(hash, sizeof(hash), nm, nmlen, algo, iter,
2868                 salt, saltlen);
2869         if(!hlen) return 0;
2870         /* b32 encode */
2871         if(*hashnmlen < hlen*2+1+z->namelen) /* approx b32 as hexb16 */
2872                 return 0;
2873         ret = sldns_b32_ntop_extended_hex(hash, hlen, (char*)(hashname+1),
2874                 (*hashnmlen)-1);
2875         if(ret<1)
2876                 return 0;
2877         hashname[0] = (uint8_t)ret;
2878         ret++;
2879         if((*hashnmlen) - ret < z->namelen)
2880                 return 0;
2881         memmove(hashname+ret, z->name, z->namelen);
2882         *hashnmlen = z->namelen+(size_t)ret;
2883         return 1;
2884 }
2885
2886 /** Find the datanode that covers the nsec3hash-name */
2887 static struct auth_data*
2888 az_nsec3_findnode(struct auth_zone* z, uint8_t* hashnm, size_t hashnmlen)
2889 {
2890         struct query_info qinfo;
2891         struct auth_data* node;
2892         int node_exact;
2893         qinfo.qclass = 0;
2894         qinfo.qtype = 0;
2895         qinfo.qname = hashnm;
2896         qinfo.qname_len = hashnmlen;
2897         /* because canonical ordering and b32 nsec3 ordering are the same.
2898          * this is a good lookup to find the nsec3 name. */
2899         az_find_domain(z, &qinfo, &node_exact, &node);
2900         /* but we may have to skip non-nsec3 nodes */
2901         /* this may be a lot, the way to speed that up is to have a
2902          * separate nsec3 tree with nsec3 nodes */
2903         while(node && (rbnode_type*)node != RBTREE_NULL &&
2904                 !az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2905                 node = (struct auth_data*)rbtree_previous(&node->node);
2906         }
2907         if((rbnode_type*)node == RBTREE_NULL)
2908                 node = NULL;
2909         return node;
2910 }
2911
2912 /** Find cover for hashed(nm, nmlen) (or NULL) */
2913 static struct auth_data*
2914 az_nsec3_find_cover(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2915         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2916 {
2917         struct auth_data* node;
2918         uint8_t hname[LDNS_MAX_DOMAINLEN];
2919         size_t hlen = sizeof(hname);
2920         if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2921                 salt, saltlen))
2922                 return NULL;
2923         node = az_nsec3_findnode(z, hname, hlen);
2924         if(node)
2925                 return node;
2926         /* we did not find any, perhaps because the NSEC3 hash is before
2927          * the first hash, we have to find the 'last hash' in the zone */
2928         node = (struct auth_data*)rbtree_last(&z->data);
2929         while(node && (rbnode_type*)node != RBTREE_NULL &&
2930                 !az_domain_rrset(node, LDNS_RR_TYPE_NSEC3)) {
2931                 node = (struct auth_data*)rbtree_previous(&node->node);
2932         }
2933         if((rbnode_type*)node == RBTREE_NULL)
2934                 node = NULL;
2935         return node;
2936 }
2937
2938 /** Find exact match for hashed(nm, nmlen) NSEC3 record or NULL */
2939 static struct auth_data*
2940 az_nsec3_find_exact(struct auth_zone* z, uint8_t* nm, size_t nmlen,
2941         int algo, size_t iter, uint8_t* salt, size_t saltlen)
2942 {
2943         struct auth_data* node;
2944         uint8_t hname[LDNS_MAX_DOMAINLEN];
2945         size_t hlen = sizeof(hname);
2946         if(!az_nsec3_hashname(z, hname, &hlen, nm, nmlen, algo, iter,
2947                 salt, saltlen))
2948                 return NULL;
2949         node = az_find_name(z, hname, hlen);
2950         if(az_domain_rrset(node, LDNS_RR_TYPE_NSEC3))
2951                 return node;
2952         return NULL;
2953 }
2954
2955 /** Return nextcloser name (as a ref into the qname).  This is one label
2956  * more than the cenm (cename must be a suffix of qname) */
2957 static void
2958 az_nsec3_get_nextcloser(uint8_t* cenm, uint8_t* qname, size_t qname_len,
2959         uint8_t** nx, size_t* nxlen)
2960 {
2961         int celabs = dname_count_labels(cenm);
2962         int qlabs = dname_count_labels(qname);
2963         int strip = qlabs - celabs -1;
2964         log_assert(dname_strict_subdomain(qname, qlabs, cenm, celabs));
2965         *nx = qname;
2966         *nxlen = qname_len;
2967         if(strip>0)
2968                 dname_remove_labels(nx, nxlen, strip);
2969 }
2970
2971 /** Find the closest encloser that has exact NSEC3.
2972  * updated cenm to the new name. If it went up no-exact-ce is true. */
2973 static struct auth_data*
2974 az_nsec3_find_ce(struct auth_zone* z, uint8_t** cenm, size_t* cenmlen,
2975         int* no_exact_ce, int algo, size_t iter, uint8_t* salt, size_t saltlen)
2976 {
2977         struct auth_data* node;
2978         while((node = az_nsec3_find_exact(z, *cenm, *cenmlen,
2979                 algo, iter, salt, saltlen)) == NULL) {
2980                 if(*cenmlen == z->namelen) {
2981                         /* next step up would take us out of the zone. fail */
2982                         return NULL;
2983                 }
2984                 *no_exact_ce = 1;
2985                 dname_remove_label(cenm, cenmlen);
2986         }
2987         return node;
2988 }
2989
2990 /* Insert NSEC3 record in authority section, if NULL does nothing */
2991 static int
2992 az_nsec3_insert(struct auth_zone* z, struct regional* region,
2993         struct dns_msg* msg, struct auth_data* node)
2994 {
2995         struct auth_rrset* nsec3;
2996         if(!node) return 1; /* no node, skip this */
2997         nsec3 = az_domain_rrset(node, LDNS_RR_TYPE_NSEC3);
2998         if(!nsec3) return 1; /* if no nsec3 RR, skip it */
2999         if(!msg_add_rrset_ns(z, region, msg, node, nsec3)) return 0;
3000         return 1;
3001 }
3002
3003 /** add NSEC3 records to the zone for the nsec3 proof.
3004  * Specify with the flags with parts of the proof are required.
3005  * the ce is the exact matching name (for notype) but also delegation points.
3006  * qname is the one where the nextcloser name can be derived from.
3007  * If NSEC3 is not properly there (in the zone) nothing is added.
3008  * always enabled: include nsec3 proving about the Closest Encloser.
3009  *      that is an exact match that should exist for it.
3010  *      If that does not exist, a higher exact match + nxproof is enabled
3011  *      (for some sort of opt-out empty nonterminal cases).
3012  * nodataproof: search for exact match and include that instead.
3013  * ceproof: include ce proof NSEC3 (omitted for wildcard replies).
3014  * nxproof: include denial of the qname.
3015  * wcproof: include denial of wildcard (wildcard.ce).
3016  */
3017 static int
3018 az_add_nsec3_proof(struct auth_zone* z, struct regional* region,
3019         struct dns_msg* msg, uint8_t* cenm, size_t cenmlen, uint8_t* qname,
3020         size_t qname_len, int nodataproof, int ceproof, int nxproof,
3021         int wcproof)
3022 {
3023         int algo;
3024         size_t iter, saltlen;
3025         uint8_t* salt;
3026         int no_exact_ce = 0;
3027         struct auth_data* node;
3028
3029         /* find parameters of nsec3 proof */
3030         if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen))
3031                 return 1; /* no nsec3 */
3032         if(nodataproof) {
3033                 /* see if the node has a hash of itself for the nodata
3034                  * proof nsec3, this has to be an exact match nsec3. */
3035                 struct auth_data* match;
3036                 match = az_nsec3_find_exact(z, qname, qname_len, algo,
3037                         iter, salt, saltlen);
3038                 if(match) {
3039                         if(!az_nsec3_insert(z, region, msg, match))
3040                                 return 0;
3041                         /* only nodata NSEC3 needed, no CE or others. */
3042                         return 1;
3043                 }
3044         }
3045         /* find ce that has an NSEC3 */
3046         if(ceproof) {
3047                 node = az_nsec3_find_ce(z, &cenm, &cenmlen, &no_exact_ce,
3048                         algo, iter, salt, saltlen);
3049                 if(no_exact_ce) nxproof = 1;
3050                 if(!az_nsec3_insert(z, region, msg, node))
3051                         return 0;
3052         }
3053
3054         if(nxproof) {
3055                 uint8_t* nx;
3056                 size_t nxlen;
3057                 /* create nextcloser domain name */
3058                 az_nsec3_get_nextcloser(cenm, qname, qname_len, &nx, &nxlen);
3059                 /* find nsec3 that matches or covers it */
3060                 node = az_nsec3_find_cover(z, nx, nxlen, algo, iter, salt,
3061                         saltlen);
3062                 if(!az_nsec3_insert(z, region, msg, node))
3063                         return 0;
3064         }
3065         if(wcproof) {
3066                 /* create wildcard name *.ce */
3067                 uint8_t wc[LDNS_MAX_DOMAINLEN];
3068                 size_t wclen;
3069                 if(cenmlen+2 > sizeof(wc))
3070                         return 0; /* result would be too long */
3071                 wc[0] = 1; /* length of wildcard label */
3072                 wc[1] = (uint8_t)'*'; /* wildcard label */
3073                 memmove(wc+2, cenm, cenmlen);
3074                 wclen = cenmlen+2;
3075                 /* find nsec3 that matches or covers it */
3076                 node = az_nsec3_find_cover(z, wc, wclen, algo, iter, salt,
3077                         saltlen);
3078                 if(!az_nsec3_insert(z, region, msg, node))
3079                         return 0;
3080         }
3081         return 1;
3082 }
3083
3084 /** generate answer for positive answer */
3085 static int
3086 az_generate_positive_answer(struct auth_zone* z, struct regional* region,
3087         struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
3088 {
3089         if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3090         /* see if we want additional rrs */
3091         if(rrset->type == LDNS_RR_TYPE_MX) {
3092                 if(!az_add_additionals_from(z, region, msg, rrset, 2))
3093                         return 0;
3094         } else if(rrset->type == LDNS_RR_TYPE_SRV) {
3095                 if(!az_add_additionals_from(z, region, msg, rrset, 6))
3096                         return 0;
3097         } else if(rrset->type == LDNS_RR_TYPE_NS) {
3098                 if(!az_add_additionals_from(z, region, msg, rrset, 0))
3099                         return 0;
3100         }
3101         return 1;
3102 }
3103
3104 /** generate answer for type ANY answer */
3105 static int
3106 az_generate_any_answer(struct auth_zone* z, struct regional* region,
3107         struct dns_msg* msg, struct auth_data* node)
3108 {
3109         struct auth_rrset* rrset;
3110         int added = 0;
3111         /* add a couple (at least one) RRs */
3112         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_SOA)) != NULL) {
3113                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3114                 added++;
3115         }
3116         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_MX)) != NULL) {
3117                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3118                 added++;
3119         }
3120         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_A)) != NULL) {
3121                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3122                 added++;
3123         }
3124         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_AAAA)) != NULL) {
3125                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3126                 added++;
3127         }
3128         if(added == 0 && node && node->rrsets) {
3129                 if(!msg_add_rrset_an(z, region, msg, node,
3130                         node->rrsets)) return 0;
3131         }
3132         return 1;
3133 }
3134
3135 /** follow cname chain and add more data to the answer section */
3136 static int
3137 follow_cname_chain(struct auth_zone* z, uint16_t qtype,
3138         struct regional* region, struct dns_msg* msg,
3139         struct packed_rrset_data* d)
3140 {
3141         int maxchain = 0;
3142         /* see if we can add the target of the CNAME into the answer */
3143         while(maxchain++ < MAX_CNAME_CHAIN) {
3144                 struct auth_data* node;
3145                 struct auth_rrset* rrset;
3146                 size_t clen;
3147                 /* d has cname rdata */
3148                 if(d->count == 0) break; /* no CNAME */
3149                 if(d->rr_len[0] < 2+1) break; /* too small */
3150                 if((clen=dname_valid(d->rr_data[0]+2, d->rr_len[0]-2))==0)
3151                         break; /* malformed */
3152                 if(!dname_subdomain_c(d->rr_data[0]+2, z->name))
3153                         break; /* target out of zone */
3154                 if((node = az_find_name(z, d->rr_data[0]+2, clen))==NULL)
3155                         break; /* no such target name */
3156                 if((rrset=az_domain_rrset(node, qtype))!=NULL) {
3157                         /* done we found the target */
3158                         if(!msg_add_rrset_an(z, region, msg, node, rrset))
3159                                 return 0;
3160                         break;
3161                 }
3162                 if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME))==NULL)
3163                         break; /* no further CNAME chain, notype */
3164                 if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3165                 d = rrset->data;
3166         }
3167         return 1;
3168 }
3169
3170 /** generate answer for cname answer */
3171 static int
3172 az_generate_cname_answer(struct auth_zone* z, struct query_info* qinfo,
3173         struct regional* region, struct dns_msg* msg,
3174         struct auth_data* node, struct auth_rrset* rrset)
3175 {
3176         if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
3177         if(!rrset) return 1;
3178         if(!follow_cname_chain(z, qinfo->qtype, region, msg, rrset->data))
3179                 return 0;
3180         return 1;
3181 }
3182
3183 /** generate answer for notype answer */
3184 static int
3185 az_generate_notype_answer(struct auth_zone* z, struct regional* region,
3186         struct dns_msg* msg, struct auth_data* node)
3187 {
3188         struct auth_rrset* rrset;
3189         if(!az_add_negative_soa(z, region, msg)) return 0;
3190         /* DNSSEC denial NSEC */
3191         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_NSEC))!=NULL) {
3192                 if(!msg_add_rrset_ns(z, region, msg, node, rrset)) return 0;
3193         } else if(node) {
3194                 /* DNSSEC denial NSEC3 */
3195                 if(!az_add_nsec3_proof(z, region, msg, node->name,
3196                         node->namelen, msg->qinfo.qname,
3197                         msg->qinfo.qname_len, 1, 1, 0, 0))
3198                         return 0;
3199         }
3200         return 1;
3201 }
3202
3203 /** generate answer for referral answer */
3204 static int
3205 az_generate_referral_answer(struct auth_zone* z, struct regional* region,
3206         struct dns_msg* msg, struct auth_data* ce, struct auth_rrset* rrset)
3207 {
3208         struct auth_rrset* ds, *nsec;
3209         /* turn off AA flag, referral is nonAA because it leaves the zone */
3210         log_assert(ce);
3211         msg->rep->flags &= ~BIT_AA;
3212         if(!msg_add_rrset_ns(z, region, msg, ce, rrset)) return 0;
3213         /* add DS or deny it */
3214         if((ds=az_domain_rrset(ce, LDNS_RR_TYPE_DS))!=NULL) {
3215                 if(!msg_add_rrset_ns(z, region, msg, ce, ds)) return 0;
3216         } else {
3217                 /* deny the DS */
3218                 if((nsec=az_domain_rrset(ce, LDNS_RR_TYPE_NSEC))!=NULL) {
3219                         if(!msg_add_rrset_ns(z, region, msg, ce, nsec))
3220                                 return 0;
3221                 } else {
3222                         if(!az_add_nsec3_proof(z, region, msg, ce->name,
3223                                 ce->namelen, msg->qinfo.qname,
3224                                 msg->qinfo.qname_len, 1, 1, 0, 0))
3225                                 return 0;
3226                 }
3227         }
3228         /* add additional rrs for type NS */
3229         if(!az_add_additionals_from(z, region, msg, rrset, 0)) return 0;
3230         return 1;
3231 }
3232
3233 /** generate answer for DNAME answer */
3234 static int
3235 az_generate_dname_answer(struct auth_zone* z, struct query_info* qinfo,
3236         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3237         struct auth_rrset* rrset)
3238 {
3239         log_assert(ce);
3240         /* add the DNAME and then a CNAME */
3241         if(!msg_add_rrset_an(z, region, msg, ce, rrset)) return 0;
3242         if(!add_synth_cname(z, qinfo->qname, qinfo->qname_len, region,
3243                 msg, ce, rrset)) return 0;
3244         if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_YXDOMAIN)
3245                 return 1;
3246         if(msg->rep->rrset_count == 0 ||
3247                 !msg->rep->rrsets[msg->rep->rrset_count-1])
3248                 return 0;
3249         if(!follow_cname_chain(z, qinfo->qtype, region, msg,
3250                 (struct packed_rrset_data*)msg->rep->rrsets[
3251                 msg->rep->rrset_count-1]->entry.data))
3252                 return 0;
3253         return 1;
3254 }
3255
3256 /** generate answer for wildcard answer */
3257 static int
3258 az_generate_wildcard_answer(struct auth_zone* z, struct query_info* qinfo,
3259         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3260         struct auth_data* wildcard, struct auth_data* node)
3261 {
3262         struct auth_rrset* rrset, *nsec;
3263         int insert_ce = 0;
3264         if((rrset=az_domain_rrset(wildcard, qinfo->qtype)) != NULL) {
3265                 /* wildcard has type, add it */
3266                 if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
3267                         return 0;
3268                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3269                         msg->qinfo.qname_len, 1);
3270         } else if((rrset=az_domain_rrset(wildcard, LDNS_RR_TYPE_CNAME))!=NULL) {
3271                 /* wildcard has cname instead, do that */
3272                 if(!msg_add_rrset_an(z, region, msg, wildcard, rrset))
3273                         return 0;
3274                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3275                         msg->qinfo.qname_len, 1);
3276                 if(!follow_cname_chain(z, qinfo->qtype, region, msg,
3277                         rrset->data))
3278                         return 0;
3279         } else if(qinfo->qtype == LDNS_RR_TYPE_ANY && wildcard->rrsets) {
3280                 /* add ANY rrsets from wildcard node */
3281                 if(!az_generate_any_answer(z, region, msg, wildcard))
3282                         return 0;
3283                 az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3284                         msg->qinfo.qname_len, 1);
3285         } else {
3286                 /* wildcard has nodata, notype answer */
3287                 /* call other notype routine for dnssec notype denials */
3288                 if(!az_generate_notype_answer(z, region, msg, wildcard))
3289                         return 0;
3290                 /* because the notype, there is no positive data with an
3291                  * RRSIG that indicates the wildcard position.  Thus the
3292                  * wildcard qname denial needs to have a CE nsec3. */
3293                 insert_ce = 1;
3294         }
3295
3296         /* ce and node for dnssec denial of wildcard original name */
3297         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3298                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3299         } else if(ce) {
3300                 uint8_t* wildup = wildcard->name;
3301                 size_t wilduplen= wildcard->namelen;
3302                 dname_remove_label(&wildup, &wilduplen);
3303                 if(!az_add_nsec3_proof(z, region, msg, wildup,
3304                         wilduplen, msg->qinfo.qname,
3305                         msg->qinfo.qname_len, 0, insert_ce, 1, 0))
3306                         return 0;
3307         }
3308
3309         /* fixup name of wildcard from *.zone to qname, use already allocated
3310          * pointer to msg qname */
3311         az_change_dnames(msg, wildcard->name, msg->qinfo.qname,
3312                 msg->qinfo.qname_len, 0);
3313         return 1;
3314 }
3315
3316 /** generate answer for nxdomain answer */
3317 static int
3318 az_generate_nxdomain_answer(struct auth_zone* z, struct regional* region,
3319         struct dns_msg* msg, struct auth_data* ce, struct auth_data* node)
3320 {
3321         struct auth_rrset* nsec;
3322         msg->rep->flags |= LDNS_RCODE_NXDOMAIN;
3323         if(!az_add_negative_soa(z, region, msg)) return 0;
3324         if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
3325                 if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
3326                 if(ce && !az_nsec_wildcard_denial(z, region, msg, ce->name,
3327                         ce->namelen)) return 0;
3328         } else if(ce) {
3329                 if(!az_add_nsec3_proof(z, region, msg, ce->name,
3330                         ce->namelen, msg->qinfo.qname,
3331                         msg->qinfo.qname_len, 0, 1, 1, 1))
3332                         return 0;
3333         }
3334         return 1;
3335 }
3336
3337 /** Create answers when an exact match exists for the domain name */
3338 static int
3339 az_generate_answer_with_node(struct auth_zone* z, struct query_info* qinfo,
3340         struct regional* region, struct dns_msg* msg, struct auth_data* node)
3341 {
3342         struct auth_rrset* rrset;
3343         /* positive answer, rrset we are looking for exists */
3344         if((rrset=az_domain_rrset(node, qinfo->qtype)) != NULL) {
3345                 return az_generate_positive_answer(z, region, msg, node, rrset);
3346         }
3347         /* CNAME? */
3348         if((rrset=az_domain_rrset(node, LDNS_RR_TYPE_CNAME)) != NULL) {
3349                 return az_generate_cname_answer(z, qinfo, region, msg,
3350                         node, rrset);
3351         }
3352         /* type ANY ? */
3353         if(qinfo->qtype == LDNS_RR_TYPE_ANY) {
3354                 return az_generate_any_answer(z, region, msg, node);
3355         }
3356         /* NOERROR/NODATA (no such type at domain name) */
3357         return az_generate_notype_answer(z, region, msg, node);
3358 }
3359
3360 /** Generate answer without an existing-node that we can use.
3361  * So it'll be a referral, DNAME or nxdomain */
3362 static int
3363 az_generate_answer_nonexistnode(struct auth_zone* z, struct query_info* qinfo,
3364         struct regional* region, struct dns_msg* msg, struct auth_data* ce,
3365         struct auth_rrset* rrset, struct auth_data* node)
3366 {
3367         struct auth_data* wildcard;
3368
3369         /* we do not have an exact matching name (that exists) */
3370         /* see if we have a NS or DNAME in the ce */
3371         if(ce && rrset && rrset->type == LDNS_RR_TYPE_NS) {
3372                 return az_generate_referral_answer(z, region, msg, ce, rrset);
3373         }
3374         if(ce && rrset && rrset->type == LDNS_RR_TYPE_DNAME) {
3375                 return az_generate_dname_answer(z, qinfo, region, msg, ce,
3376                         rrset);
3377         }
3378         /* if there is an empty nonterminal, wildcard and nxdomain don't
3379          * happen, it is a notype answer */
3380         if(az_empty_nonterminal(z, qinfo, node)) {
3381                 return az_generate_notype_answer(z, region, msg, node);
3382         }
3383         /* see if we have a wildcard under the ce */
3384         if((wildcard=az_find_wildcard(z, qinfo, ce)) != NULL) {
3385                 return az_generate_wildcard_answer(z, qinfo, region, msg,
3386                         ce, wildcard, node);
3387         }
3388         /* generate nxdomain answer */
3389         return az_generate_nxdomain_answer(z, region, msg, ce, node);
3390 }
3391
3392 /** Lookup answer in a zone. */
3393 static int
3394 auth_zone_generate_answer(struct auth_zone* z, struct query_info* qinfo,
3395         struct regional* region, struct dns_msg** msg, int* fallback)
3396 {
3397         struct auth_data* node, *ce;
3398         struct auth_rrset* rrset;
3399         int node_exact, node_exists;
3400         /* does the zone want fallback in case of failure? */
3401         *fallback = z->fallback_enabled;
3402         if(!(*msg=msg_create(region, qinfo))) return 0;
3403
3404         /* lookup if there is a matching domain name for the query */
3405         az_find_domain(z, qinfo, &node_exact, &node);
3406
3407         /* see if node exists for generating answers from (i.e. not glue and
3408          * obscured by NS or DNAME or NSEC3-only), and also return the
3409          * closest-encloser from that, closest node that should be used
3410          * to generate answers from that is above the query */
3411         node_exists = az_find_ce(z, qinfo, node, node_exact, &ce, &rrset);
3412
3413         if(verbosity >= VERB_ALGO) {
3414                 char zname[256], qname[256], nname[256], cename[256],
3415                         tpstr[32], rrstr[32];
3416                 sldns_wire2str_dname_buf(qinfo->qname, qinfo->qname_len, qname,
3417                         sizeof(qname));
3418                 sldns_wire2str_type_buf(qinfo->qtype, tpstr, sizeof(tpstr));
3419                 sldns_wire2str_dname_buf(z->name, z->namelen, zname,
3420                         sizeof(zname));
3421                 if(node)
3422                         sldns_wire2str_dname_buf(node->name, node->namelen,
3423                                 nname, sizeof(nname));
3424                 else    snprintf(nname, sizeof(nname), "NULL");
3425                 if(ce)
3426                         sldns_wire2str_dname_buf(ce->name, ce->namelen,
3427                                 cename, sizeof(cename));
3428                 else    snprintf(cename, sizeof(cename), "NULL");
3429                 if(rrset) sldns_wire2str_type_buf(rrset->type, rrstr,
3430                         sizeof(rrstr));
3431                 else    snprintf(rrstr, sizeof(rrstr), "NULL");
3432                 log_info("auth_zone %s query %s %s, domain %s %s %s, "
3433                         "ce %s, rrset %s", zname, qname, tpstr, nname,
3434                         (node_exact?"exact":"notexact"),
3435                         (node_exists?"exist":"notexist"), cename, rrstr);
3436         }
3437
3438         if(node_exists) {
3439                 /* the node is fine, generate answer from node */
3440                 return az_generate_answer_with_node(z, qinfo, region, *msg,
3441                         node);
3442         }
3443         return az_generate_answer_nonexistnode(z, qinfo, region, *msg,
3444                 ce, rrset, node);
3445 }
3446
3447 int auth_zones_lookup(struct auth_zones* az, struct query_info* qinfo,
3448         struct regional* region, struct dns_msg** msg, int* fallback,
3449         uint8_t* dp_nm, size_t dp_nmlen)
3450 {
3451         int r;
3452         struct auth_zone* z;
3453         /* find the zone that should contain the answer. */
3454         lock_rw_rdlock(&az->lock);
3455         z = auth_zone_find(az, dp_nm, dp_nmlen, qinfo->qclass);
3456         if(!z) {
3457                 lock_rw_unlock(&az->lock);
3458                 /* no auth zone, fallback to internet */
3459                 *fallback = 1;
3460                 return 0;
3461         }
3462         lock_rw_rdlock(&z->lock);
3463         lock_rw_unlock(&az->lock);
3464
3465         /* if not for upstream queries, fallback */
3466         if(!z->for_upstream) {
3467                 lock_rw_unlock(&z->lock);
3468                 *fallback = 1;
3469                 return 0;
3470         }
3471         if(z->zone_expired) {
3472                 *fallback = z->fallback_enabled;
3473                 lock_rw_unlock(&z->lock);
3474                 return 0;
3475         }
3476         /* see what answer that zone would generate */
3477         r = auth_zone_generate_answer(z, qinfo, region, msg, fallback);
3478         lock_rw_unlock(&z->lock);
3479         return r;
3480 }
3481
3482 /** encode auth answer */
3483 static void
3484 auth_answer_encode(struct query_info* qinfo, struct module_env* env,
3485         struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3486         struct regional* temp, struct dns_msg* msg)
3487 {
3488         uint16_t udpsize;
3489         udpsize = edns->udp_size;
3490         edns->edns_version = EDNS_ADVERTISED_VERSION;
3491         edns->udp_size = EDNS_ADVERTISED_SIZE;
3492         edns->ext_rcode = 0;
3493         edns->bits &= EDNS_DO;
3494
3495         if(!inplace_cb_reply_local_call(env, qinfo, NULL, msg->rep,
3496                 (int)FLAGS_GET_RCODE(msg->rep->flags), edns, repinfo, temp, env->now_tv)
3497                 || !reply_info_answer_encode(qinfo, msg->rep,
3498                 *(uint16_t*)sldns_buffer_begin(buf),
3499                 sldns_buffer_read_u16_at(buf, 2),
3500                 buf, 0, 0, temp, udpsize, edns,
3501                 (int)(edns->bits&EDNS_DO), 0)) {
3502                 error_encode(buf, (LDNS_RCODE_SERVFAIL|BIT_AA), qinfo,
3503                         *(uint16_t*)sldns_buffer_begin(buf),
3504                         sldns_buffer_read_u16_at(buf, 2), edns);
3505         }
3506 }
3507
3508 /** encode auth error answer */
3509 static void
3510 auth_error_encode(struct query_info* qinfo, struct module_env* env,
3511         struct edns_data* edns, struct comm_reply* repinfo, sldns_buffer* buf,
3512         struct regional* temp, int rcode)
3513 {
3514         edns->edns_version = EDNS_ADVERTISED_VERSION;
3515         edns->udp_size = EDNS_ADVERTISED_SIZE;
3516         edns->ext_rcode = 0;
3517         edns->bits &= EDNS_DO;
3518
3519         if(!inplace_cb_reply_local_call(env, qinfo, NULL, NULL,
3520                 rcode, edns, repinfo, temp, env->now_tv))
3521                 edns->opt_list_inplace_cb_out = NULL;
3522         error_encode(buf, rcode|BIT_AA, qinfo,
3523                 *(uint16_t*)sldns_buffer_begin(buf),
3524                 sldns_buffer_read_u16_at(buf, 2), edns);
3525 }
3526
3527 int auth_zones_answer(struct auth_zones* az, struct module_env* env,
3528         struct query_info* qinfo, struct edns_data* edns,
3529         struct comm_reply* repinfo, struct sldns_buffer* buf, struct regional* temp)
3530 {
3531         struct dns_msg* msg = NULL;
3532         struct auth_zone* z;
3533         int r;
3534         int fallback = 0;
3535
3536         lock_rw_rdlock(&az->lock);
3537         if(!az->have_downstream) {
3538                 /* no downstream auth zones */
3539                 lock_rw_unlock(&az->lock);
3540                 return 0;
3541         }
3542         if(qinfo->qtype == LDNS_RR_TYPE_DS) {
3543                 uint8_t* delname = qinfo->qname;
3544                 size_t delnamelen = qinfo->qname_len;
3545                 dname_remove_label(&delname, &delnamelen);
3546                 z = auth_zones_find_zone(az, delname, delnamelen,
3547                         qinfo->qclass);
3548         } else {
3549                 z = auth_zones_find_zone(az, qinfo->qname, qinfo->qname_len,
3550                         qinfo->qclass);
3551         }
3552         if(!z) {
3553                 /* no zone above it */
3554                 lock_rw_unlock(&az->lock);
3555                 return 0;
3556         }
3557         lock_rw_rdlock(&z->lock);
3558         lock_rw_unlock(&az->lock);
3559         if(!z->for_downstream) {
3560                 lock_rw_unlock(&z->lock);
3561                 return 0;
3562         }
3563         if(z->zone_expired) {
3564                 if(z->fallback_enabled) {
3565                         lock_rw_unlock(&z->lock);
3566                         return 0;
3567                 }
3568                 lock_rw_unlock(&z->lock);
3569                 lock_rw_wrlock(&az->lock);
3570                 az->num_query_down++;
3571                 lock_rw_unlock(&az->lock);
3572                 auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3573                         LDNS_RCODE_SERVFAIL);
3574                 return 1;
3575         }
3576
3577         /* answer it from zone z */
3578         r = auth_zone_generate_answer(z, qinfo, temp, &msg, &fallback);
3579         lock_rw_unlock(&z->lock);
3580         if(!r && fallback) {
3581                 /* fallback to regular answering (recursive) */
3582                 return 0;
3583         }
3584         lock_rw_wrlock(&az->lock);
3585         az->num_query_down++;
3586         lock_rw_unlock(&az->lock);
3587
3588         /* encode answer */
3589         if(!r)
3590                 auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
3591                         LDNS_RCODE_SERVFAIL);
3592         else    auth_answer_encode(qinfo, env, edns, repinfo, buf, temp, msg);
3593
3594         return 1;
3595 }
3596
3597 int auth_zones_can_fallback(struct auth_zones* az, uint8_t* nm, size_t nmlen,
3598         uint16_t dclass)
3599 {
3600         int r;
3601         struct auth_zone* z;
3602         lock_rw_rdlock(&az->lock);
3603         z = auth_zone_find(az, nm, nmlen, dclass);
3604         if(!z) {
3605                 lock_rw_unlock(&az->lock);
3606                 /* no such auth zone, fallback */
3607                 return 1;
3608         }
3609         lock_rw_rdlock(&z->lock);
3610         lock_rw_unlock(&az->lock);
3611         r = z->fallback_enabled || (!z->for_upstream);
3612         lock_rw_unlock(&z->lock);
3613         return r;
3614 }
3615
3616 int
3617 auth_zone_parse_notify_serial(sldns_buffer* pkt, uint32_t *serial)
3618 {
3619         struct query_info q;
3620         uint16_t rdlen;
3621         memset(&q, 0, sizeof(q));
3622         sldns_buffer_set_position(pkt, 0);
3623         if(!query_info_parse(&q, pkt)) return 0;
3624         if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0) return 0;
3625         /* skip name of RR in answer section */
3626         if(sldns_buffer_remaining(pkt) < 1) return 0;
3627         if(pkt_dname_len(pkt) == 0) return 0;
3628         /* check type */
3629         if(sldns_buffer_remaining(pkt) < 10 /* type,class,ttl,rdatalen*/)
3630                 return 0;
3631         if(sldns_buffer_read_u16(pkt) != LDNS_RR_TYPE_SOA) return 0;
3632         sldns_buffer_skip(pkt, 2); /* class */
3633         sldns_buffer_skip(pkt, 4); /* ttl */
3634         rdlen = sldns_buffer_read_u16(pkt); /* rdatalen */
3635         if(sldns_buffer_remaining(pkt) < rdlen) return 0;
3636         if(rdlen < 22) return 0; /* bad soa length */
3637         sldns_buffer_skip(pkt, (ssize_t)(rdlen-20));
3638         *serial = sldns_buffer_read_u32(pkt);
3639         /* return true when has serial in answer section */
3640         return 1;
3641 }
3642
3643 /** see if addr appears in the list */
3644 static int
3645 addr_in_list(struct auth_addr* list, struct sockaddr_storage* addr,
3646         socklen_t addrlen)
3647 {
3648         struct auth_addr* p;
3649         for(p=list; p; p=p->next) {
3650                 if(sockaddr_cmp_addr(addr, addrlen, &p->addr, p->addrlen)==0)
3651                         return 1;
3652         }
3653         return 0;
3654 }
3655
3656 /** check if an address matches a master specification (or one of its
3657  * addresses in the addr list) */
3658 static int
3659 addr_matches_master(struct auth_master* master, struct sockaddr_storage* addr,
3660         socklen_t addrlen, struct auth_master** fromhost)
3661 {
3662         struct sockaddr_storage a;
3663         socklen_t alen = 0;
3664         int net = 0;
3665         if(addr_in_list(master->list, addr, addrlen)) {
3666                 *fromhost = master;
3667                 return 1;
3668         }
3669         /* compare address (but not port number, that is the destination
3670          * port of the master, the port number of the received notify is
3671          * allowed to by any port on that master) */
3672         if(extstrtoaddr(master->host, &a, &alen) &&
3673                 sockaddr_cmp_addr(addr, addrlen, &a, alen)==0) {
3674                 *fromhost = master;
3675                 return 1;
3676         }
3677         /* prefixes, addr/len, like 10.0.0.0/8 */
3678         /* not http and has a / and there is one / */
3679         if(master->allow_notify && !master->http &&
3680                 strchr(master->host, '/') != NULL &&
3681                 strchr(master->host, '/') == strrchr(master->host, '/') &&
3682                 netblockstrtoaddr(master->host, UNBOUND_DNS_PORT, &a, &alen,
3683                 &net) && alen == addrlen) {
3684                 if(addr_in_common(addr, (addr_is_ip6(addr, addrlen)?128:32),
3685                         &a, net, alen) >= net) {
3686                         *fromhost = NULL; /* prefix does not have destination
3687                                 to send the probe or transfer with */
3688                         return 1; /* matches the netblock */
3689                 }
3690         }
3691         return 0;
3692 }
3693
3694 /** check access list for notifies */
3695 static int
3696 az_xfr_allowed_notify(struct auth_xfer* xfr, struct sockaddr_storage* addr,
3697         socklen_t addrlen, struct auth_master** fromhost)
3698 {
3699         struct auth_master* p;
3700         for(p=xfr->allow_notify_list; p; p=p->next) {
3701                 if(addr_matches_master(p, addr, addrlen, fromhost)) {
3702                         return 1;
3703                 }
3704         }
3705         return 0;
3706 }
3707
3708 /** see if the serial means the zone has to be updated, i.e. the serial
3709  * is newer than the zone serial, or we have no zone */
3710 static int
3711 xfr_serial_means_update(struct auth_xfer* xfr, uint32_t serial)
3712 {
3713         if(!xfr->have_zone)
3714                 return 1; /* no zone, anything is better */
3715         if(xfr->zone_expired)
3716                 return 1; /* expired, the sent serial is better than expired
3717                         data */
3718         if(compare_serial(xfr->serial, serial) < 0)
3719                 return 1; /* our serial is smaller than the sent serial,
3720                         the data is newer, fetch it */
3721         return 0;
3722 }
3723
3724 /** note notify serial, updates the notify information in the xfr struct */
3725 static void
3726 xfr_note_notify_serial(struct auth_xfer* xfr, int has_serial, uint32_t serial)
3727 {
3728         if(xfr->notify_received && xfr->notify_has_serial && has_serial) {
3729                 /* see if this serial is newer */
3730                 if(compare_serial(xfr->notify_serial, serial) < 0)
3731                         xfr->notify_serial = serial;
3732         } else if(xfr->notify_received && xfr->notify_has_serial &&
3733                 !has_serial) {
3734                 /* remove serial, we have notify without serial */
3735                 xfr->notify_has_serial = 0;
3736                 xfr->notify_serial = 0;
3737         } else if(xfr->notify_received && !xfr->notify_has_serial) {
3738                 /* we already have notify without serial, keep it
3739                  * that way; no serial check when current operation
3740                  * is done */
3741         } else {
3742                 xfr->notify_received = 1;
3743                 xfr->notify_has_serial = has_serial;
3744                 xfr->notify_serial = serial;
3745         }
3746 }
3747
3748 /** process a notify serial, start new probe or note serial. xfr is locked */
3749 static void
3750 xfr_process_notify(struct auth_xfer* xfr, struct module_env* env,
3751         int has_serial, uint32_t serial, struct auth_master* fromhost)
3752 {
3753         /* if the serial of notify is older than we have, don't fetch
3754          * a zone, we already have it */
3755         if(has_serial && !xfr_serial_means_update(xfr, serial)) {
3756                 lock_basic_unlock(&xfr->lock);
3757                 return;
3758         }
3759         /* start new probe with this addr src, or note serial */
3760         if(!xfr_start_probe(xfr, env, fromhost)) {
3761                 /* not started because already in progress, note the serial */
3762                 xfr_note_notify_serial(xfr, has_serial, serial);
3763                 lock_basic_unlock(&xfr->lock);
3764         }
3765         /* successful end of start_probe unlocked xfr->lock */
3766 }
3767
3768 int auth_zones_notify(struct auth_zones* az, struct module_env* env,
3769         uint8_t* nm, size_t nmlen, uint16_t dclass,
3770         struct sockaddr_storage* addr, socklen_t addrlen, int has_serial,
3771         uint32_t serial, int* refused)
3772 {
3773         struct auth_xfer* xfr;
3774         struct auth_master* fromhost = NULL;
3775         /* see which zone this is */
3776         lock_rw_rdlock(&az->lock);
3777         xfr = auth_xfer_find(az, nm, nmlen, dclass);
3778         if(!xfr) {
3779                 lock_rw_unlock(&az->lock);
3780                 /* no such zone, refuse the notify */
3781                 *refused = 1;
3782                 return 0;
3783         }
3784         lock_basic_lock(&xfr->lock);
3785         lock_rw_unlock(&az->lock);
3786
3787         /* check access list for notifies */
3788         if(!az_xfr_allowed_notify(xfr, addr, addrlen, &fromhost)) {
3789                 lock_basic_unlock(&xfr->lock);
3790                 /* notify not allowed, refuse the notify */
3791                 *refused = 1;
3792                 return 0;
3793         }
3794
3795         /* process the notify */
3796         xfr_process_notify(xfr, env, has_serial, serial, fromhost);
3797         return 1;
3798 }
3799
3800 int auth_zones_startprobesequence(struct auth_zones* az,
3801         struct module_env* env, uint8_t* nm, size_t nmlen, uint16_t dclass)
3802 {
3803         struct auth_xfer* xfr;
3804         lock_rw_rdlock(&az->lock);
3805         xfr = auth_xfer_find(az, nm, nmlen, dclass);
3806         if(!xfr) {
3807                 lock_rw_unlock(&az->lock);
3808                 return 0;
3809         }
3810         lock_basic_lock(&xfr->lock);
3811         lock_rw_unlock(&az->lock);
3812
3813         xfr_process_notify(xfr, env, 0, 0, NULL);
3814         return 1;
3815 }
3816
3817 /** set a zone expired */
3818 static void
3819 auth_xfer_set_expired(struct auth_xfer* xfr, struct module_env* env,
3820         int expired)
3821 {
3822         struct auth_zone* z;
3823
3824         /* expire xfr */
3825         lock_basic_lock(&xfr->lock);
3826         xfr->zone_expired = expired;
3827         lock_basic_unlock(&xfr->lock);
3828
3829         /* find auth_zone */
3830         lock_rw_rdlock(&env->auth_zones->lock);
3831         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
3832                 xfr->dclass);
3833         if(!z) {
3834                 lock_rw_unlock(&env->auth_zones->lock);
3835                 return;
3836         }
3837         lock_rw_wrlock(&z->lock);
3838         lock_rw_unlock(&env->auth_zones->lock);
3839
3840         /* expire auth_zone */
3841         z->zone_expired = expired;
3842         lock_rw_unlock(&z->lock);
3843 }
3844
3845 /** find master (from notify or probe) in list of masters */
3846 static struct auth_master*
3847 find_master_by_host(struct auth_master* list, char* host)
3848 {
3849         struct auth_master* p;
3850         for(p=list; p; p=p->next) {
3851                 if(strcmp(p->host, host) == 0)
3852                         return p;
3853         }
3854         return NULL;
3855 }
3856
3857 /** delete the looked up auth_addrs for all the masters in the list */
3858 static void
3859 xfr_masterlist_free_addrs(struct auth_master* list)
3860 {
3861         struct auth_master* m;
3862         for(m=list; m; m=m->next) {
3863                 if(m->list) {
3864                         auth_free_master_addrs(m->list);
3865                         m->list = NULL;
3866                 }
3867         }
3868 }
3869
3870 /** copy a list of auth_addrs */
3871 static struct auth_addr*
3872 auth_addr_list_copy(struct auth_addr* source)
3873 {
3874         struct auth_addr* list = NULL, *last = NULL;
3875         struct auth_addr* p;
3876         for(p=source; p; p=p->next) {
3877                 struct auth_addr* a = (struct auth_addr*)memdup(p, sizeof(*p));
3878                 if(!a) {
3879                         log_err("malloc failure");
3880                         auth_free_master_addrs(list);
3881                         return NULL;
3882                 }
3883                 a->next = NULL;
3884                 if(last) last->next = a;
3885                 if(!list) list = a;
3886                 last = a;
3887         }
3888         return list;
3889 }
3890
3891 /** copy a master to a new structure, NULL on alloc failure */
3892 static struct auth_master*
3893 auth_master_copy(struct auth_master* o)
3894 {
3895         struct auth_master* m;
3896         if(!o) return NULL;
3897         m = (struct auth_master*)memdup(o, sizeof(*o));
3898         if(!m) {
3899                 log_err("malloc failure");
3900                 return NULL;
3901         }
3902         m->next = NULL;
3903         if(m->host) {
3904                 m->host = strdup(m->host);
3905                 if(!m->host) {
3906                         free(m);
3907                         log_err("malloc failure");
3908                         return NULL;
3909                 }
3910         }
3911         if(m->file) {
3912                 m->file = strdup(m->file);
3913                 if(!m->file) {
3914                         free(m->host);
3915                         free(m);
3916                         log_err("malloc failure");
3917                         return NULL;
3918                 }
3919         }
3920         if(m->list) {
3921                 m->list = auth_addr_list_copy(m->list);
3922                 if(!m->list) {
3923                         free(m->file);
3924                         free(m->host);
3925                         free(m);
3926                         return NULL;
3927                 }
3928         }
3929         return m;
3930 }
3931
3932 /** copy the master addresses from the task_probe lookups to the allow_notify
3933  * list of masters */
3934 static void
3935 probe_copy_masters_for_allow_notify(struct auth_xfer* xfr)
3936 {
3937         struct auth_master* list = NULL, *last = NULL;
3938         struct auth_master* p;
3939         /* build up new list with copies */
3940         for(p = xfr->task_probe->masters; p; p=p->next) {
3941                 struct auth_master* m = auth_master_copy(p);
3942                 if(!m) {
3943                         auth_free_masters(list);
3944                         /* failed because of malloc failure, use old list */
3945                         return;
3946                 }
3947                 m->next = NULL;
3948                 if(last) last->next = m;
3949                 if(!list) list = m;
3950                 last = m;
3951         }
3952         /* success, replace list */
3953         auth_free_masters(xfr->allow_notify_list);
3954         xfr->allow_notify_list = list;
3955 }
3956
3957 /** start the lookups for task_transfer */
3958 static void
3959 xfr_transfer_start_lookups(struct auth_xfer* xfr)
3960 {
3961         /* delete all the looked up addresses in the list */
3962         xfr->task_transfer->scan_addr = NULL;
3963         xfr_masterlist_free_addrs(xfr->task_transfer->masters);
3964
3965         /* start lookup at the first master */
3966         xfr->task_transfer->lookup_target = xfr->task_transfer->masters;
3967         xfr->task_transfer->lookup_aaaa = 0;
3968 }
3969
3970 /** move to the next lookup of hostname for task_transfer */
3971 static void
3972 xfr_transfer_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
3973 {
3974         if(!xfr->task_transfer->lookup_target)
3975                 return; /* already at end of list */
3976         if(!xfr->task_transfer->lookup_aaaa && env->cfg->do_ip6) {
3977                 /* move to lookup AAAA */
3978                 xfr->task_transfer->lookup_aaaa = 1;
3979                 return;
3980         }
3981         xfr->task_transfer->lookup_target =
3982                 xfr->task_transfer->lookup_target->next;
3983         xfr->task_transfer->lookup_aaaa = 0;
3984         if(!env->cfg->do_ip4 && xfr->task_transfer->lookup_target!=NULL)
3985                 xfr->task_transfer->lookup_aaaa = 1;
3986 }
3987
3988 /** start the lookups for task_probe */
3989 static void
3990 xfr_probe_start_lookups(struct auth_xfer* xfr)
3991 {
3992         /* delete all the looked up addresses in the list */
3993         xfr->task_probe->scan_addr = NULL;
3994         xfr_masterlist_free_addrs(xfr->task_probe->masters);
3995
3996         /* start lookup at the first master */
3997         xfr->task_probe->lookup_target = xfr->task_probe->masters;
3998         xfr->task_probe->lookup_aaaa = 0;
3999 }
4000
4001 /** move to the next lookup of hostname for task_probe */
4002 static void
4003 xfr_probe_move_to_next_lookup(struct auth_xfer* xfr, struct module_env* env)
4004 {
4005         if(!xfr->task_probe->lookup_target)
4006                 return; /* already at end of list */
4007         if(!xfr->task_probe->lookup_aaaa && env->cfg->do_ip6) {
4008                 /* move to lookup AAAA */
4009                 xfr->task_probe->lookup_aaaa = 1;
4010                 return;
4011         }
4012         xfr->task_probe->lookup_target = xfr->task_probe->lookup_target->next;
4013         xfr->task_probe->lookup_aaaa = 0;
4014         if(!env->cfg->do_ip4 && xfr->task_probe->lookup_target!=NULL)
4015                 xfr->task_probe->lookup_aaaa = 1;
4016 }
4017
4018 /** start the iteration of the task_transfer list of masters */
4019 static void
4020 xfr_transfer_start_list(struct auth_xfer* xfr, struct auth_master* spec)
4021 {
4022         if(spec) {
4023                 xfr->task_transfer->scan_specific = find_master_by_host(
4024                         xfr->task_transfer->masters, spec->host);
4025                 if(xfr->task_transfer->scan_specific) {
4026                         xfr->task_transfer->scan_target = NULL;
4027                         xfr->task_transfer->scan_addr = NULL;
4028                         if(xfr->task_transfer->scan_specific->list)
4029                                 xfr->task_transfer->scan_addr =
4030                                         xfr->task_transfer->scan_specific->list;
4031                         return;
4032                 }
4033         }
4034         /* no specific (notified) host to scan */
4035         xfr->task_transfer->scan_specific = NULL;
4036         xfr->task_transfer->scan_addr = NULL;
4037         /* pick up first scan target */
4038         xfr->task_transfer->scan_target = xfr->task_transfer->masters;
4039         if(xfr->task_transfer->scan_target && xfr->task_transfer->
4040                 scan_target->list)
4041                 xfr->task_transfer->scan_addr =
4042                         xfr->task_transfer->scan_target->list;
4043 }
4044
4045 /** start the iteration of the task_probe list of masters */
4046 static void
4047 xfr_probe_start_list(struct auth_xfer* xfr, struct auth_master* spec)
4048 {
4049         if(spec) {
4050                 xfr->task_probe->scan_specific = find_master_by_host(
4051                         xfr->task_probe->masters, spec->host);
4052                 if(xfr->task_probe->scan_specific) {
4053                         xfr->task_probe->scan_target = NULL;
4054                         xfr->task_probe->scan_addr = NULL;
4055                         if(xfr->task_probe->scan_specific->list)
4056                                 xfr->task_probe->scan_addr =
4057                                         xfr->task_probe->scan_specific->list;
4058                         return;
4059                 }
4060         }
4061         /* no specific (notified) host to scan */
4062         xfr->task_probe->scan_specific = NULL;
4063         xfr->task_probe->scan_addr = NULL;
4064         /* pick up first scan target */
4065         xfr->task_probe->scan_target = xfr->task_probe->masters;
4066         if(xfr->task_probe->scan_target && xfr->task_probe->scan_target->list)
4067                 xfr->task_probe->scan_addr =
4068                         xfr->task_probe->scan_target->list;
4069 }
4070
4071 /** pick up the master that is being scanned right now, task_transfer */
4072 static struct auth_master*
4073 xfr_transfer_current_master(struct auth_xfer* xfr)
4074 {
4075         if(xfr->task_transfer->scan_specific)
4076                 return xfr->task_transfer->scan_specific;
4077         return xfr->task_transfer->scan_target;
4078 }
4079
4080 /** pick up the master that is being scanned right now, task_probe */
4081 static struct auth_master*
4082 xfr_probe_current_master(struct auth_xfer* xfr)
4083 {
4084         if(xfr->task_probe->scan_specific)
4085                 return xfr->task_probe->scan_specific;
4086         return xfr->task_probe->scan_target;
4087 }
4088
4089 /** true if at end of list, task_transfer */
4090 static int
4091 xfr_transfer_end_of_list(struct auth_xfer* xfr)
4092 {
4093         return !xfr->task_transfer->scan_specific &&
4094                 !xfr->task_transfer->scan_target;
4095 }
4096
4097 /** true if at end of list, task_probe */
4098 static int
4099 xfr_probe_end_of_list(struct auth_xfer* xfr)
4100 {
4101         return !xfr->task_probe->scan_specific && !xfr->task_probe->scan_target;
4102 }
4103
4104 /** move to next master in list, task_transfer */
4105 static void
4106 xfr_transfer_nextmaster(struct auth_xfer* xfr)
4107 {
4108         if(!xfr->task_transfer->scan_specific &&
4109                 !xfr->task_transfer->scan_target)
4110                 return;
4111         if(xfr->task_transfer->scan_addr) {
4112                 xfr->task_transfer->scan_addr =
4113                         xfr->task_transfer->scan_addr->next;
4114                 if(xfr->task_transfer->scan_addr)
4115                         return;
4116         }
4117         if(xfr->task_transfer->scan_specific) {
4118                 xfr->task_transfer->scan_specific = NULL;
4119                 xfr->task_transfer->scan_target = xfr->task_transfer->masters;
4120                 if(xfr->task_transfer->scan_target && xfr->task_transfer->
4121                         scan_target->list)
4122                         xfr->task_transfer->scan_addr =
4123                                 xfr->task_transfer->scan_target->list;
4124                 return;
4125         }
4126         if(!xfr->task_transfer->scan_target)
4127                 return;
4128         xfr->task_transfer->scan_target = xfr->task_transfer->scan_target->next;
4129         if(xfr->task_transfer->scan_target && xfr->task_transfer->
4130                 scan_target->list)
4131                 xfr->task_transfer->scan_addr =
4132                         xfr->task_transfer->scan_target->list;
4133         return;
4134 }
4135
4136 /** move to next master in list, task_probe */
4137 static void
4138 xfr_probe_nextmaster(struct auth_xfer* xfr)
4139 {
4140         if(!xfr->task_probe->scan_specific && !xfr->task_probe->scan_target)
4141                 return;
4142         if(xfr->task_probe->scan_addr) {
4143                 xfr->task_probe->scan_addr = xfr->task_probe->scan_addr->next;
4144                 if(xfr->task_probe->scan_addr)
4145                         return;
4146         }
4147         if(xfr->task_probe->scan_specific) {
4148                 xfr->task_probe->scan_specific = NULL;
4149                 xfr->task_probe->scan_target = xfr->task_probe->masters;
4150                 if(xfr->task_probe->scan_target && xfr->task_probe->
4151                         scan_target->list)
4152                         xfr->task_probe->scan_addr =
4153                                 xfr->task_probe->scan_target->list;
4154                 return;
4155         }
4156         if(!xfr->task_probe->scan_target)
4157                 return;
4158         xfr->task_probe->scan_target = xfr->task_probe->scan_target->next;
4159         if(xfr->task_probe->scan_target && xfr->task_probe->
4160                 scan_target->list)
4161                 xfr->task_probe->scan_addr =
4162                         xfr->task_probe->scan_target->list;
4163         return;
4164 }
4165
4166 /** create SOA probe packet for xfr */
4167 static void
4168 xfr_create_soa_probe_packet(struct auth_xfer* xfr, sldns_buffer* buf,
4169         uint16_t id)
4170 {
4171         struct query_info qinfo;
4172
4173         memset(&qinfo, 0, sizeof(qinfo));
4174         qinfo.qname = xfr->name;
4175         qinfo.qname_len = xfr->namelen;
4176         qinfo.qtype = LDNS_RR_TYPE_SOA;
4177         qinfo.qclass = xfr->dclass;
4178         qinfo_query_encode(buf, &qinfo);
4179         sldns_buffer_write_u16_at(buf, 0, id);
4180 }
4181
4182 /** create IXFR/AXFR packet for xfr */
4183 static void
4184 xfr_create_ixfr_packet(struct auth_xfer* xfr, sldns_buffer* buf, uint16_t id,
4185         struct auth_master* master)
4186 {
4187         struct query_info qinfo;
4188         uint32_t serial;
4189         int have_zone;
4190         have_zone = xfr->have_zone;
4191         serial = xfr->serial;
4192
4193         memset(&qinfo, 0, sizeof(qinfo));
4194         qinfo.qname = xfr->name;
4195         qinfo.qname_len = xfr->namelen;
4196         xfr->task_transfer->got_xfr_serial = 0;
4197         xfr->task_transfer->rr_scan_num = 0;
4198         xfr->task_transfer->incoming_xfr_serial = 0;
4199         xfr->task_transfer->on_ixfr_is_axfr = 0;
4200         xfr->task_transfer->on_ixfr = 1;
4201         qinfo.qtype = LDNS_RR_TYPE_IXFR;
4202         if(!have_zone || xfr->task_transfer->ixfr_fail || !master->ixfr) {
4203                 qinfo.qtype = LDNS_RR_TYPE_AXFR;
4204                 xfr->task_transfer->ixfr_fail = 0;
4205                 xfr->task_transfer->on_ixfr = 0;
4206         }
4207
4208         qinfo.qclass = xfr->dclass;
4209         qinfo_query_encode(buf, &qinfo);
4210         sldns_buffer_write_u16_at(buf, 0, id);
4211
4212         /* append serial for IXFR */
4213         if(qinfo.qtype == LDNS_RR_TYPE_IXFR) {
4214                 size_t end = sldns_buffer_limit(buf);
4215                 sldns_buffer_clear(buf);
4216                 sldns_buffer_set_position(buf, end);
4217                 /* auth section count 1 */
4218                 sldns_buffer_write_u16_at(buf, LDNS_NSCOUNT_OFF, 1);
4219                 /* write SOA */
4220                 sldns_buffer_write_u8(buf, 0xC0); /* compressed ptr to qname */
4221                 sldns_buffer_write_u8(buf, 0x0C);
4222                 sldns_buffer_write_u16(buf, LDNS_RR_TYPE_SOA);
4223                 sldns_buffer_write_u16(buf, qinfo.qclass);
4224                 sldns_buffer_write_u32(buf, 0); /* ttl */
4225                 sldns_buffer_write_u16(buf, 22); /* rdata length */
4226                 sldns_buffer_write_u8(buf, 0); /* . */
4227                 sldns_buffer_write_u8(buf, 0); /* . */
4228                 sldns_buffer_write_u32(buf, serial); /* serial */
4229                 sldns_buffer_write_u32(buf, 0); /* refresh */
4230                 sldns_buffer_write_u32(buf, 0); /* retry */
4231                 sldns_buffer_write_u32(buf, 0); /* expire */
4232                 sldns_buffer_write_u32(buf, 0); /* minimum */
4233                 sldns_buffer_flip(buf);
4234         }
4235 }
4236
4237 /** check if returned packet is OK */
4238 static int
4239 check_packet_ok(sldns_buffer* pkt, uint16_t qtype, struct auth_xfer* xfr,
4240         uint32_t* serial)
4241 {
4242         /* parse to see if packet worked, valid reply */
4243
4244         /* check serial number of SOA */
4245         if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE)
4246                 return 0;
4247
4248         /* check ID */
4249         if(LDNS_ID_WIRE(sldns_buffer_begin(pkt)) != xfr->task_probe->id)
4250                 return 0;
4251
4252         /* check flag bits and rcode */
4253         if(!LDNS_QR_WIRE(sldns_buffer_begin(pkt)))
4254                 return 0;
4255         if(LDNS_OPCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_PACKET_QUERY)
4256                 return 0;
4257         if(LDNS_RCODE_WIRE(sldns_buffer_begin(pkt)) != LDNS_RCODE_NOERROR)
4258                 return 0;
4259
4260         /* check qname */
4261         if(LDNS_QDCOUNT(sldns_buffer_begin(pkt)) != 1)
4262                 return 0;
4263         sldns_buffer_skip(pkt, LDNS_HEADER_SIZE);
4264         if(sldns_buffer_remaining(pkt) < xfr->namelen)
4265                 return 0;
4266         if(query_dname_compare(sldns_buffer_current(pkt), xfr->name) != 0)
4267                 return 0;
4268         sldns_buffer_skip(pkt, (ssize_t)xfr->namelen);
4269
4270         /* check qtype, qclass */
4271         if(sldns_buffer_remaining(pkt) < 4)
4272                 return 0;
4273         if(sldns_buffer_read_u16(pkt) != qtype)
4274                 return 0;
4275         if(sldns_buffer_read_u16(pkt) != xfr->dclass)
4276                 return 0;
4277
4278         if(serial) {
4279                 uint16_t rdlen;
4280                 /* read serial number, from answer section SOA */
4281                 if(LDNS_ANCOUNT(sldns_buffer_begin(pkt)) == 0)
4282                         return 0;
4283                 /* read from first record SOA record */
4284                 if(sldns_buffer_remaining(pkt) < 1)
4285                         return 0;
4286                 if(dname_pkt_compare(pkt, sldns_buffer_current(pkt),
4287                         xfr->name) != 0)
4288                         return 0;
4289                 if(!pkt_dname_len(pkt))
4290                         return 0;
4291                 /* type, class, ttl, rdatalen */
4292                 if(sldns_buffer_remaining(pkt) < 4+4+2)
4293                         return 0;
4294                 if(sldns_buffer_read_u16(pkt) != qtype)
4295                         return 0;
4296                 if(sldns_buffer_read_u16(pkt) != xfr->dclass)
4297                         return 0;
4298                 sldns_buffer_skip(pkt, 4); /* ttl */
4299                 rdlen = sldns_buffer_read_u16(pkt);
4300                 if(sldns_buffer_remaining(pkt) < rdlen)
4301                         return 0;
4302                 if(sldns_buffer_remaining(pkt) < 1)
4303                         return 0;
4304                 if(!pkt_dname_len(pkt)) /* soa name */
4305                         return 0;
4306                 if(sldns_buffer_remaining(pkt) < 1)
4307                         return 0;
4308                 if(!pkt_dname_len(pkt)) /* soa name */
4309                         return 0;
4310                 if(sldns_buffer_remaining(pkt) < 20)
4311                         return 0;
4312                 *serial = sldns_buffer_read_u32(pkt);
4313         }
4314         return 1;
4315 }
4316
4317 /** read one line from chunks into buffer at current position */
4318 static int
4319 chunkline_get_line(struct auth_chunk** chunk, size_t* chunk_pos,
4320         sldns_buffer* buf)
4321 {
4322         int readsome = 0;
4323         while(*chunk) {
4324                 /* more text in this chunk? */
4325                 if(*chunk_pos < (*chunk)->len) {
4326                         readsome = 1;
4327                         while(*chunk_pos < (*chunk)->len) {
4328                                 char c = (char)((*chunk)->data[*chunk_pos]);
4329                                 (*chunk_pos)++;
4330                                 if(sldns_buffer_remaining(buf) < 2) {
4331                                         /* buffer too short */
4332                                         verbose(VERB_ALGO, "http chunkline, "
4333                                                 "line too long");
4334                                         return 0;
4335                                 }
4336                                 sldns_buffer_write_u8(buf, (uint8_t)c);
4337                                 if(c == '\n') {
4338                                         /* we are done */
4339                                         return 1;
4340                                 }
4341                         }
4342                 }
4343                 /* move to next chunk */
4344                 *chunk = (*chunk)->next;
4345                 *chunk_pos = 0;
4346         }
4347         /* no more text */
4348         if(readsome) return 1;
4349         return 0;
4350 }
4351
4352 /** count number of open and closed parenthesis in a chunkline */
4353 static int
4354 chunkline_count_parens(sldns_buffer* buf, size_t start)
4355 {
4356         size_t end = sldns_buffer_position(buf);
4357         size_t i;
4358         int count = 0;
4359         int squote = 0, dquote = 0;
4360         for(i=start; i<end; i++) {
4361                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4362                 if(squote && c != '\'') continue;
4363                 if(dquote && c != '"') continue;
4364                 if(c == '"')
4365                         dquote = !dquote; /* skip quoted part */
4366                 else if(c == '\'')
4367                         squote = !squote; /* skip quoted part */
4368                 else if(c == '(')
4369                         count ++;
4370                 else if(c == ')')
4371                         count --;
4372                 else if(c == ';') {
4373                         /* rest is a comment */
4374                         return count;
4375                 }
4376         }
4377         return count;
4378 }
4379
4380 /** remove trailing ;... comment from a line in the chunkline buffer */
4381 static void
4382 chunkline_remove_trailcomment(sldns_buffer* buf, size_t start)
4383 {
4384         size_t end = sldns_buffer_position(buf);
4385         size_t i;
4386         int squote = 0, dquote = 0;
4387         for(i=start; i<end; i++) {
4388                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4389                 if(squote && c != '\'') continue;
4390                 if(dquote && c != '"') continue;
4391                 if(c == '"')
4392                         dquote = !dquote; /* skip quoted part */
4393                 else if(c == '\'')
4394                         squote = !squote; /* skip quoted part */
4395                 else if(c == ';') {
4396                         /* rest is a comment */
4397                         sldns_buffer_set_position(buf, i);
4398                         return;
4399                 }
4400         }
4401         /* nothing to remove */
4402 }
4403
4404 /** see if a chunkline is a comment line (or empty line) */
4405 static int
4406 chunkline_is_comment_line_or_empty(sldns_buffer* buf)
4407 {
4408         size_t i, end = sldns_buffer_limit(buf);
4409         for(i=0; i<end; i++) {
4410                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4411                 if(c == ';')
4412                         return 1; /* comment */
4413                 else if(c != ' ' && c != '\t' && c != '\r' && c != '\n')
4414                         return 0; /* not a comment */
4415         }
4416         return 1; /* empty */
4417 }
4418
4419 /** find a line with ( ) collated */
4420 static int
4421 chunkline_get_line_collated(struct auth_chunk** chunk, size_t* chunk_pos,
4422         sldns_buffer* buf)
4423 {
4424         size_t pos;
4425         int parens = 0;
4426         sldns_buffer_clear(buf);
4427         pos = sldns_buffer_position(buf);
4428         if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4429                 if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4430                         sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4431                 else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4432                 sldns_buffer_flip(buf);
4433                 return 0;
4434         }
4435         parens += chunkline_count_parens(buf, pos);
4436         while(parens > 0) {
4437                 chunkline_remove_trailcomment(buf, pos);
4438                 pos = sldns_buffer_position(buf);
4439                 if(!chunkline_get_line(chunk, chunk_pos, buf)) {
4440                         if(sldns_buffer_position(buf) < sldns_buffer_limit(buf))
4441                                 sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4442                         else sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf)-1, 0);
4443                         sldns_buffer_flip(buf);
4444                         return 0;
4445                 }
4446                 parens += chunkline_count_parens(buf, pos);
4447         }
4448
4449         if(sldns_buffer_remaining(buf) < 1) {
4450                 verbose(VERB_ALGO, "http chunkline: "
4451                         "line too long");
4452                 return 0;
4453         }
4454         sldns_buffer_write_u8_at(buf, sldns_buffer_position(buf), 0);
4455         sldns_buffer_flip(buf);
4456         return 1;
4457 }
4458
4459 /** process $ORIGIN for http */
4460 static int
4461 http_parse_origin(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4462 {
4463         char* line = (char*)sldns_buffer_begin(buf);
4464         if(strncmp(line, "$ORIGIN", 7) == 0 &&
4465                 isspace((unsigned char)line[7])) {
4466                 int s;
4467                 pstate->origin_len = sizeof(pstate->origin);
4468                 s = sldns_str2wire_dname_buf(sldns_strip_ws(line+8),
4469                         pstate->origin, &pstate->origin_len);
4470                 if(s) pstate->origin_len = 0;
4471                 return 1;
4472         }
4473         return 0;
4474 }
4475
4476 /** process $TTL for http */
4477 static int
4478 http_parse_ttl(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4479 {
4480         char* line = (char*)sldns_buffer_begin(buf);
4481         if(strncmp(line, "$TTL", 4) == 0 &&
4482                 isspace((unsigned char)line[4])) {
4483                 const char* end = NULL;
4484                 pstate->default_ttl = sldns_str2period(
4485                         sldns_strip_ws(line+5), &end);
4486                 return 1;
4487         }
4488         return 0;
4489 }
4490
4491 /** find noncomment RR line in chunks, collates lines if ( ) format */
4492 static int
4493 chunkline_non_comment_RR(struct auth_chunk** chunk, size_t* chunk_pos,
4494         sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4495 {
4496         while(chunkline_get_line_collated(chunk, chunk_pos, buf)) {
4497                 if(chunkline_is_comment_line_or_empty(buf)) {
4498                         /* a comment, go to next line */
4499                         continue;
4500                 }
4501                 if(http_parse_origin(buf, pstate)) {
4502                         continue; /* $ORIGIN has been handled */
4503                 }
4504                 if(http_parse_ttl(buf, pstate)) {
4505                         continue; /* $TTL has been handled */
4506                 }
4507                 return 1;
4508         }
4509         /* no noncomments, fail */
4510         return 0;
4511 }
4512
4513 /** check syntax of chunklist zonefile, parse first RR, return false on
4514  * failure and return a string in the scratch buffer (first RR string)
4515  * on failure. */
4516 static int
4517 http_zonefile_syntax_check(struct auth_xfer* xfr, sldns_buffer* buf)
4518 {
4519         uint8_t rr[LDNS_RR_BUF_SIZE];
4520         size_t rr_len, dname_len = 0;
4521         struct sldns_file_parse_state pstate;
4522         struct auth_chunk* chunk;
4523         size_t chunk_pos;
4524         int e;
4525         memset(&pstate, 0, sizeof(pstate));
4526         pstate.default_ttl = 3600;
4527         if(xfr->namelen < sizeof(pstate.origin)) {
4528                 pstate.origin_len = xfr->namelen;
4529                 memmove(pstate.origin, xfr->name, xfr->namelen);
4530         }
4531         chunk = xfr->task_transfer->chunks_first;
4532         chunk_pos = 0;
4533         if(!chunkline_non_comment_RR(&chunk, &chunk_pos, buf, &pstate)) {
4534                 return 0;
4535         }
4536         rr_len = sizeof(rr);
4537         e=sldns_str2wire_rr_buf((char*)sldns_buffer_begin(buf), rr, &rr_len,
4538                 &dname_len, pstate.default_ttl,
4539                 pstate.origin_len?pstate.origin:NULL, pstate.origin_len,
4540                 pstate.prev_rr_len?pstate.prev_rr:NULL, pstate.prev_rr_len);
4541         if(e != 0) {
4542                 log_err("parse failure on first RR[%d]: %s",
4543                         LDNS_WIREPARSE_OFFSET(e),
4544                         sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)));
4545                 return 0;
4546         }
4547         /* check that class is correct */
4548         if(sldns_wirerr_get_class(rr, rr_len, dname_len) != xfr->dclass) {
4549                 log_err("parse failure: first record in downloaded zonefile "
4550                         "from wrong RR class");
4551                 return 0;
4552         }
4553         return 1;
4554 }
4555
4556 /** sum sizes of chunklist */
4557 static size_t
4558 chunklist_sum(struct auth_chunk* list)
4559 {
4560         struct auth_chunk* p;
4561         size_t s = 0;
4562         for(p=list; p; p=p->next) {
4563                 s += p->len;
4564         }
4565         return s;
4566 }
4567
4568 /** remove newlines from collated line */
4569 static void
4570 chunkline_newline_removal(sldns_buffer* buf)
4571 {
4572         size_t i, end=sldns_buffer_limit(buf);
4573         for(i=0; i<end; i++) {
4574                 char c = (char)sldns_buffer_read_u8_at(buf, i);
4575                 if(c == '\n' && i==end-1) {
4576                         sldns_buffer_write_u8_at(buf, i, 0);
4577                         sldns_buffer_set_limit(buf, end-1);
4578                         return;
4579                 }
4580                 if(c == '\n')
4581                         sldns_buffer_write_u8_at(buf, i, (uint8_t)' ');
4582         }
4583 }
4584
4585 /** for http download, parse and add RR to zone */
4586 static int
4587 http_parse_add_rr(struct auth_xfer* xfr, struct auth_zone* z,
4588         sldns_buffer* buf, struct sldns_file_parse_state* pstate)
4589 {
4590         uint8_t rr[LDNS_RR_BUF_SIZE];
4591         size_t rr_len, dname_len = 0;
4592         int e;
4593         char* line = (char*)sldns_buffer_begin(buf);
4594         rr_len = sizeof(rr);
4595         e = sldns_str2wire_rr_buf(line, rr, &rr_len, &dname_len,
4596                 pstate->default_ttl,
4597                 pstate->origin_len?pstate->origin:NULL, pstate->origin_len,
4598                 pstate->prev_rr_len?pstate->prev_rr:NULL, pstate->prev_rr_len);
4599         if(e != 0) {
4600                 log_err("%s/%s parse failure RR[%d]: %s in '%s'",
4601                         xfr->task_transfer->master->host,
4602                         xfr->task_transfer->master->file,
4603                         LDNS_WIREPARSE_OFFSET(e),
4604                         sldns_get_errorstr_parse(LDNS_WIREPARSE_ERROR(e)),
4605                         line);
4606                 return 0;
4607         }
4608         if(rr_len == 0)
4609                 return 1; /* empty line or so */
4610
4611         /* set prev */
4612         if(dname_len < sizeof(pstate->prev_rr)) {
4613                 memmove(pstate->prev_rr, rr, dname_len);
4614                 pstate->prev_rr_len = dname_len;
4615         }
4616
4617         return az_insert_rr(z, rr, rr_len, dname_len, NULL);
4618 }
4619
4620 /** RR list iterator, returns RRs from answer section one by one from the
4621  * dns packets in the chunklist */
4622 static void
4623 chunk_rrlist_start(struct auth_xfer* xfr, struct auth_chunk** rr_chunk,
4624         int* rr_num, size_t* rr_pos)
4625 {
4626         *rr_chunk = xfr->task_transfer->chunks_first;
4627         *rr_num = 0;
4628         *rr_pos = 0;
4629 }
4630
4631 /** RR list iterator, see if we are at the end of the list */
4632 static int
4633 chunk_rrlist_end(struct auth_chunk* rr_chunk, int rr_num)
4634 {
4635         while(rr_chunk) {
4636                 if(rr_chunk->len < LDNS_HEADER_SIZE)
4637                         return 1;
4638                 if(rr_num < (int)LDNS_ANCOUNT(rr_chunk->data))
4639                         return 0;
4640                 /* no more RRs in this chunk */
4641                 /* continue with next chunk, see if it has RRs */
4642                 rr_chunk = rr_chunk->next;
4643                 rr_num = 0;
4644         }
4645         return 1;
4646 }
4647
4648 /** RR list iterator, move to next RR */
4649 static void
4650 chunk_rrlist_gonext(struct auth_chunk** rr_chunk, int* rr_num,
4651         size_t* rr_pos, size_t rr_nextpos)
4652 {
4653         /* already at end of chunks? */
4654         if(!*rr_chunk)
4655                 return;
4656         /* move within this chunk */
4657         if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4658                 (*rr_num)+1 < (int)LDNS_ANCOUNT((*rr_chunk)->data)) {
4659                 (*rr_num) += 1;
4660                 *rr_pos = rr_nextpos;
4661                 return;
4662         }
4663         /* no more RRs in this chunk */
4664         /* continue with next chunk, see if it has RRs */
4665         if(*rr_chunk)
4666                 *rr_chunk = (*rr_chunk)->next;
4667         while(*rr_chunk) {
4668                 *rr_num = 0;
4669                 *rr_pos = 0;
4670                 if((*rr_chunk)->len >= LDNS_HEADER_SIZE &&
4671                         LDNS_ANCOUNT((*rr_chunk)->data) > 0) {
4672                         return;
4673                 }
4674                 *rr_chunk = (*rr_chunk)->next;
4675         }
4676 }
4677
4678 /** RR iterator, get current RR information, false on parse error */
4679 static int
4680 chunk_rrlist_get_current(struct auth_chunk* rr_chunk, int rr_num,
4681         size_t rr_pos, uint8_t** rr_dname, uint16_t* rr_type,
4682         uint16_t* rr_class, uint32_t* rr_ttl, uint16_t* rr_rdlen,
4683         uint8_t** rr_rdata, size_t* rr_nextpos)
4684 {
4685         sldns_buffer pkt;
4686         /* integrity checks on position */
4687         if(!rr_chunk) return 0;
4688         if(rr_chunk->len < LDNS_HEADER_SIZE) return 0;
4689         if(rr_num >= (int)LDNS_ANCOUNT(rr_chunk->data)) return 0;
4690         if(rr_pos >= rr_chunk->len) return 0;
4691
4692         /* fetch rr information */
4693         sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4694         if(rr_pos == 0) {
4695                 size_t i;
4696                 /* skip question section */
4697                 sldns_buffer_set_position(&pkt, LDNS_HEADER_SIZE);
4698                 for(i=0; i<LDNS_QDCOUNT(rr_chunk->data); i++) {
4699                         if(pkt_dname_len(&pkt) == 0) return 0;
4700                         if(sldns_buffer_remaining(&pkt) < 4) return 0;
4701                         sldns_buffer_skip(&pkt, 4); /* type and class */
4702                 }
4703         } else  {
4704                 sldns_buffer_set_position(&pkt, rr_pos);
4705         }
4706         *rr_dname = sldns_buffer_current(&pkt);
4707         if(pkt_dname_len(&pkt) == 0) return 0;
4708         if(sldns_buffer_remaining(&pkt) < 10) return 0;
4709         *rr_type = sldns_buffer_read_u16(&pkt);
4710         *rr_class = sldns_buffer_read_u16(&pkt);
4711         *rr_ttl = sldns_buffer_read_u32(&pkt);
4712         *rr_rdlen = sldns_buffer_read_u16(&pkt);
4713         if(sldns_buffer_remaining(&pkt) < (*rr_rdlen)) return 0;
4714         *rr_rdata = sldns_buffer_current(&pkt);
4715         sldns_buffer_skip(&pkt, (ssize_t)(*rr_rdlen));
4716         *rr_nextpos = sldns_buffer_position(&pkt);
4717         return 1;
4718 }
4719
4720 /** print log message where we are in parsing the zone transfer */
4721 static void
4722 log_rrlist_position(const char* label, struct auth_chunk* rr_chunk,
4723         uint8_t* rr_dname, uint16_t rr_type, size_t rr_counter)
4724 {
4725         sldns_buffer pkt;
4726         size_t dlen;
4727         uint8_t buf[256];
4728         char str[256];
4729         char typestr[32];
4730         sldns_buffer_init_frm_data(&pkt, rr_chunk->data, rr_chunk->len);
4731         sldns_buffer_set_position(&pkt, (size_t)(rr_dname -
4732                 sldns_buffer_begin(&pkt)));
4733         if((dlen=pkt_dname_len(&pkt)) == 0) return;
4734         if(dlen >= sizeof(buf)) return;
4735         dname_pkt_copy(&pkt, buf, rr_dname);
4736         dname_str(buf, str);
4737         (void)sldns_wire2str_type_buf(rr_type, typestr, sizeof(typestr));
4738         verbose(VERB_ALGO, "%s at[%d] %s %s", label, (int)rr_counter,
4739                 str, typestr);
4740 }
4741
4742 /** check that start serial is OK for ixfr. we are at rr_counter == 0,
4743  * and we are going to check rr_counter == 1 (has to be type SOA) serial */
4744 static int
4745 ixfr_start_serial(struct auth_chunk* rr_chunk, int rr_num, size_t rr_pos,
4746         uint8_t* rr_dname, uint16_t rr_type, uint16_t rr_class,
4747         uint32_t rr_ttl, uint16_t rr_rdlen, uint8_t* rr_rdata,
4748         size_t rr_nextpos, uint32_t transfer_serial, uint32_t xfr_serial)
4749 {
4750         uint32_t startserial;
4751         /* move forward on RR */
4752         chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4753         if(chunk_rrlist_end(rr_chunk, rr_num)) {
4754                 /* no second SOA */
4755                 verbose(VERB_OPS, "IXFR has no second SOA record");
4756                 return 0;
4757         }
4758         if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4759                 &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4760                 &rr_rdata, &rr_nextpos)) {
4761                 verbose(VERB_OPS, "IXFR cannot parse second SOA record");
4762                 /* failed to parse RR */
4763                 return 0;
4764         }
4765         if(rr_type != LDNS_RR_TYPE_SOA) {
4766                 verbose(VERB_OPS, "IXFR second record is not type SOA");
4767                 return 0;
4768         }
4769         if(rr_rdlen < 22) {
4770                 verbose(VERB_OPS, "IXFR, second SOA has short rdlength");
4771                 return 0; /* bad SOA rdlen */
4772         }
4773         startserial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4774         if(startserial == transfer_serial) {
4775                 /* empty AXFR, not an IXFR */
4776                 verbose(VERB_OPS, "IXFR second serial same as first");
4777                 return 0;
4778         }
4779         if(startserial != xfr_serial) {
4780                 /* wrong start serial, it does not match the serial in
4781                  * memory */
4782                 verbose(VERB_OPS, "IXFR is from serial %u to %u but %u "
4783                         "in memory, rejecting the zone transfer",
4784                         (unsigned)startserial, (unsigned)transfer_serial,
4785                         (unsigned)xfr_serial);
4786                 return 0;
4787         }
4788         /* everything OK in second SOA serial */
4789         return 1;
4790 }
4791
4792 /** apply IXFR to zone in memory. z is locked. false on failure(mallocfail) */
4793 static int
4794 apply_ixfr(struct auth_xfer* xfr, struct auth_zone* z,
4795         struct sldns_buffer* scratch_buffer)
4796 {
4797         struct auth_chunk* rr_chunk;
4798         int rr_num;
4799         size_t rr_pos;
4800         uint8_t* rr_dname, *rr_rdata;
4801         uint16_t rr_type, rr_class, rr_rdlen;
4802         uint32_t rr_ttl;
4803         size_t rr_nextpos;
4804         int have_transfer_serial = 0;
4805         uint32_t transfer_serial = 0;
4806         size_t rr_counter = 0;
4807         int delmode = 0;
4808         int softfail = 0;
4809
4810         /* start RR iterator over chunklist of packets */
4811         chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4812         while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4813                 if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4814                         &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4815                         &rr_rdata, &rr_nextpos)) {
4816                         /* failed to parse RR */
4817                         return 0;
4818                 }
4819                 if(verbosity>=7) log_rrlist_position("apply ixfr",
4820                         rr_chunk, rr_dname, rr_type, rr_counter);
4821                 /* twiddle add/del mode and check for start and end */
4822                 if(rr_counter == 0 && rr_type != LDNS_RR_TYPE_SOA)
4823                         return 0;
4824                 if(rr_counter == 1 && rr_type != LDNS_RR_TYPE_SOA) {
4825                         /* this is an AXFR returned from the IXFR master */
4826                         /* but that should already have been detected, by
4827                          * on_ixfr_is_axfr */
4828                         return 0;
4829                 }
4830                 if(rr_type == LDNS_RR_TYPE_SOA) {
4831                         uint32_t serial;
4832                         if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4833                         serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4834                         if(have_transfer_serial == 0) {
4835                                 have_transfer_serial = 1;
4836                                 transfer_serial = serial;
4837                                 delmode = 1; /* gets negated below */
4838                                 /* check second RR before going any further */
4839                                 if(!ixfr_start_serial(rr_chunk, rr_num, rr_pos,
4840                                         rr_dname, rr_type, rr_class, rr_ttl,
4841                                         rr_rdlen, rr_rdata, rr_nextpos,
4842                                         transfer_serial, xfr->serial)) {
4843                                         return 0;
4844                                 }
4845                         } else if(transfer_serial == serial) {
4846                                 have_transfer_serial++;
4847                                 if(rr_counter == 1) {
4848                                         /* empty AXFR, with SOA; SOA; */
4849                                         /* should have been detected by
4850                                          * on_ixfr_is_axfr */
4851                                         return 0;
4852                                 }
4853                                 if(have_transfer_serial == 3) {
4854                                         /* see serial three times for end */
4855                                         /* eg. IXFR:
4856                                          *  SOA 3 start
4857                                          *  SOA 1 second RR, followed by del
4858                                          *  SOA 2 followed by add
4859                                          *  SOA 2 followed by del
4860                                          *  SOA 3 followed by add
4861                                          *  SOA 3 end */
4862                                         /* ended by SOA record */
4863                                         xfr->serial = transfer_serial;
4864                                         break;
4865                                 }
4866                         }
4867                         /* twiddle add/del mode */
4868                         /* switch from delete part to add part and back again
4869                          * just before the soa, it gets deleted and added too
4870                          * this means we switch to delete mode for the final
4871                          * SOA(so skip that one) */
4872                         delmode = !delmode;
4873                 }
4874                 /* process this RR */
4875                 /* if the RR is deleted twice or added twice, then we
4876                  * softfail, and continue with the rest of the IXFR, so
4877                  * that we serve something fairly nice during the refetch */
4878                 if(verbosity>=7) log_rrlist_position((delmode?"del":"add"),
4879                         rr_chunk, rr_dname, rr_type, rr_counter);
4880                 if(delmode) {
4881                         /* delete this RR */
4882                         int nonexist = 0;
4883                         if(!az_remove_rr_decompress(z, rr_chunk->data,
4884                                 rr_chunk->len, scratch_buffer, rr_dname,
4885                                 rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4886                                 &nonexist)) {
4887                                 /* failed, malloc error or so */
4888                                 return 0;
4889                         }
4890                         if(nonexist) {
4891                                 /* it was removal of a nonexisting RR */
4892                                 if(verbosity>=4) log_rrlist_position(
4893                                         "IXFR error nonexistent RR",
4894                                         rr_chunk, rr_dname, rr_type, rr_counter);
4895                                 softfail = 1;
4896                         }
4897                 } else if(rr_counter != 0) {
4898                         /* skip first SOA RR for addition, it is added in
4899                          * the addition part near the end of the ixfr, when
4900                          * that serial is seen the second time. */
4901                         int duplicate = 0;
4902                         /* add this RR */
4903                         if(!az_insert_rr_decompress(z, rr_chunk->data,
4904                                 rr_chunk->len, scratch_buffer, rr_dname,
4905                                 rr_type, rr_class, rr_ttl, rr_rdata, rr_rdlen,
4906                                 &duplicate)) {
4907                                 /* failed, malloc error or so */
4908                                 return 0;
4909                         }
4910                         if(duplicate) {
4911                                 /* it was a duplicate */
4912                                 if(verbosity>=4) log_rrlist_position(
4913                                         "IXFR error duplicate RR",
4914                                         rr_chunk, rr_dname, rr_type, rr_counter);
4915                                 softfail = 1;
4916                         }
4917                 }
4918
4919                 rr_counter++;
4920                 chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4921         }
4922         if(softfail) {
4923                 verbose(VERB_ALGO, "IXFR did not apply cleanly, fetching full zone");
4924                 return 0;
4925         }
4926         return 1;
4927 }
4928
4929 /** apply AXFR to zone in memory. z is locked. false on failure(mallocfail) */
4930 static int
4931 apply_axfr(struct auth_xfer* xfr, struct auth_zone* z,
4932         struct sldns_buffer* scratch_buffer)
4933 {
4934         struct auth_chunk* rr_chunk;
4935         int rr_num;
4936         size_t rr_pos;
4937         uint8_t* rr_dname, *rr_rdata;
4938         uint16_t rr_type, rr_class, rr_rdlen;
4939         uint32_t rr_ttl;
4940         uint32_t serial = 0;
4941         size_t rr_nextpos;
4942         size_t rr_counter = 0;
4943         int have_end_soa = 0;
4944
4945         /* clear the data tree */
4946         traverse_postorder(&z->data, auth_data_del, NULL);
4947         rbtree_init(&z->data, &auth_data_cmp);
4948         /* clear the RPZ policies */
4949         if(z->rpz)
4950                 rpz_clear(z->rpz);
4951
4952         xfr->have_zone = 0;
4953         xfr->serial = 0;
4954
4955         /* insert all RRs in to the zone */
4956         /* insert the SOA only once, skip the last one */
4957         /* start RR iterator over chunklist of packets */
4958         chunk_rrlist_start(xfr, &rr_chunk, &rr_num, &rr_pos);
4959         while(!chunk_rrlist_end(rr_chunk, rr_num)) {
4960                 if(!chunk_rrlist_get_current(rr_chunk, rr_num, rr_pos,
4961                         &rr_dname, &rr_type, &rr_class, &rr_ttl, &rr_rdlen,
4962                         &rr_rdata, &rr_nextpos)) {
4963                         /* failed to parse RR */
4964                         return 0;
4965                 }
4966                 if(verbosity>=7) log_rrlist_position("apply_axfr",
4967                         rr_chunk, rr_dname, rr_type, rr_counter);
4968                 if(rr_type == LDNS_RR_TYPE_SOA) {
4969                         if(rr_counter != 0) {
4970                                 /* end of the axfr */
4971                                 have_end_soa = 1;
4972                                 break;
4973                         }
4974                         if(rr_rdlen < 22) return 0; /* bad SOA rdlen */
4975                         serial = sldns_read_uint32(rr_rdata+rr_rdlen-20);
4976                 }
4977
4978                 /* add this RR */
4979                 if(!az_insert_rr_decompress(z, rr_chunk->data, rr_chunk->len,
4980                         scratch_buffer, rr_dname, rr_type, rr_class, rr_ttl,
4981                         rr_rdata, rr_rdlen, NULL)) {
4982                         /* failed, malloc error or so */
4983                         return 0;
4984                 }
4985
4986                 rr_counter++;
4987                 chunk_rrlist_gonext(&rr_chunk, &rr_num, &rr_pos, rr_nextpos);
4988         }
4989         if(!have_end_soa) {
4990                 log_err("no end SOA record for AXFR");
4991                 return 0;
4992         }
4993
4994         xfr->serial = serial;
4995         xfr->have_zone = 1;
4996         return 1;
4997 }
4998
4999 /** apply HTTP to zone in memory. z is locked. false on failure(mallocfail) */
5000 static int
5001 apply_http(struct auth_xfer* xfr, struct auth_zone* z,
5002         struct sldns_buffer* scratch_buffer)
5003 {
5004         /* parse data in chunks */
5005         /* parse RR's and read into memory. ignore $INCLUDE from the
5006          * downloaded file*/
5007         struct sldns_file_parse_state pstate;
5008         struct auth_chunk* chunk;
5009         size_t chunk_pos;
5010         memset(&pstate, 0, sizeof(pstate));
5011         pstate.default_ttl = 3600;
5012         if(xfr->namelen < sizeof(pstate.origin)) {
5013                 pstate.origin_len = xfr->namelen;
5014                 memmove(pstate.origin, xfr->name, xfr->namelen);
5015         }
5016
5017         if(verbosity >= VERB_ALGO)
5018                 verbose(VERB_ALGO, "http download %s of size %d",
5019                 xfr->task_transfer->master->file,
5020                 (int)chunklist_sum(xfr->task_transfer->chunks_first));
5021         if(xfr->task_transfer->chunks_first && verbosity >= VERB_ALGO) {
5022                 char preview[1024];
5023                 if(xfr->task_transfer->chunks_first->len+1 > sizeof(preview)) {
5024                         memmove(preview, xfr->task_transfer->chunks_first->data,
5025                                 sizeof(preview)-1);
5026                         preview[sizeof(preview)-1]=0;
5027                 } else {
5028                         memmove(preview, xfr->task_transfer->chunks_first->data,
5029                                 xfr->task_transfer->chunks_first->len);
5030                         preview[xfr->task_transfer->chunks_first->len]=0;
5031                 }
5032                 log_info("auth zone http downloaded content preview: %s",
5033                         preview);
5034         }
5035
5036         /* perhaps a little syntax check before we try to apply the data? */
5037         if(!http_zonefile_syntax_check(xfr, scratch_buffer)) {
5038                 log_err("http download %s/%s does not contain a zonefile, "
5039                         "but got '%s'", xfr->task_transfer->master->host,
5040                         xfr->task_transfer->master->file,
5041                         sldns_buffer_begin(scratch_buffer));
5042                 return 0;
5043         }
5044
5045         /* clear the data tree */
5046         traverse_postorder(&z->data, auth_data_del, NULL);
5047         rbtree_init(&z->data, &auth_data_cmp);
5048         /* clear the RPZ policies */
5049         if(z->rpz)
5050                 rpz_clear(z->rpz);
5051
5052         xfr->have_zone = 0;
5053         xfr->serial = 0;
5054
5055         chunk = xfr->task_transfer->chunks_first;
5056         chunk_pos = 0;
5057         pstate.lineno = 0;
5058         while(chunkline_get_line_collated(&chunk, &chunk_pos, scratch_buffer)) {
5059                 /* process this line */
5060                 pstate.lineno++;
5061                 chunkline_newline_removal(scratch_buffer);
5062                 if(chunkline_is_comment_line_or_empty(scratch_buffer)) {
5063                         continue;
5064                 }
5065                 /* parse line and add RR */
5066                 if(http_parse_origin(scratch_buffer, &pstate)) {
5067                         continue; /* $ORIGIN has been handled */
5068                 }
5069                 if(http_parse_ttl(scratch_buffer, &pstate)) {
5070                         continue; /* $TTL has been handled */
5071                 }
5072                 if(!http_parse_add_rr(xfr, z, scratch_buffer, &pstate)) {
5073                         verbose(VERB_ALGO, "error parsing line [%s:%d] %s",
5074                                 xfr->task_transfer->master->file,
5075                                 pstate.lineno,
5076                                 sldns_buffer_begin(scratch_buffer));
5077                         return 0;
5078                 }
5079         }
5080         return 1;
5081 }
5082
5083 /** write http chunks to zonefile to create downloaded file */
5084 static int
5085 auth_zone_write_chunks(struct auth_xfer* xfr, const char* fname)
5086 {
5087         FILE* out;
5088         struct auth_chunk* p;
5089         out = fopen(fname, "w");
5090         if(!out) {
5091                 log_err("could not open %s: %s", fname, strerror(errno));
5092                 return 0;
5093         }
5094         for(p = xfr->task_transfer->chunks_first; p ; p = p->next) {
5095                 if(!write_out(out, (char*)p->data, p->len)) {
5096                         log_err("could not write http download to %s", fname);
5097                         fclose(out);
5098                         return 0;
5099                 }
5100         }
5101         fclose(out);
5102         return 1;
5103 }
5104
5105 /** write to zonefile after zone has been updated */
5106 static void
5107 xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
5108 {
5109         struct config_file* cfg = env->cfg;
5110         struct auth_zone* z;
5111         char tmpfile[1024];
5112         char* zfilename;
5113         lock_basic_unlock(&xfr->lock);
5114
5115         /* get lock again, so it is a readlock and concurrently queries
5116          * can be answered */
5117         lock_rw_rdlock(&env->auth_zones->lock);
5118         z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
5119                 xfr->dclass);
5120         if(!z) {
5121                 lock_rw_unlock(&env->auth_zones->lock);
5122                 /* the zone is gone, ignore xfr results */
5123                 lock_basic_lock(&xfr->lock);
5124                 return;
5125         }
5126         lock_rw_rdlock(&z->lock);
5127         lock_basic_lock(&xfr->lock);
5128         lock_rw_unlock(&env->auth_zones->lock);
5129
5130         if(z->zonefile == NULL || z->zonefile[0] == 0) {
5131                 lock_rw_unlock(&z->lock);
5132                 /* no write needed, no zonefile set */
5133                 return;
5134         }
5135         zfilename = z->zonefile;
5136         if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
5137                 cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
5138                 zfilename += strlen(cfg->chrootdir);
5139         if(verbosity >= VERB_ALGO) {
5140                 char nm[255+1];
5141                 dname_str(z->name, nm);
5142                 verbose(VERB_ALGO, "write zonefile %s for %s", zfilename, nm);
5143         }
5144
5145         /* write to tempfile first */
5146         if((size_t)strlen(zfilename) + 16 > sizeof(tmpfile)) {
5147                 verbose(VERB_ALGO, "tmpfilename too long, cannot update "
5148                         " zonefile %s", zfilename);
5149                 lock_rw_unlock(&z->lock);
5150                 return;
5151         }
5152         snprintf(tmpfile, sizeof(tmpfile), "%s.tmp%u", zfilename,
5153                 (unsigned)getpid());
5154         if(xfr->task_transfer->master->http) {
5155                 /* use the stored chunk list to write them */
5156                 if(!auth_zone_write_chunks(xfr, tmpfile)) {
5157                         unlink(tmpfile);
5158                         lock_rw_unlock(&z->lock);
5159                         return;
5160                 }
5161         } else if(!auth_zone_write_file(z, tmpfile)) {
5162                 unlink(tmpfile);
5163                 lock_rw_unlock(&z->lock);
5164                 return;
5165         }
5166 #ifdef UB_ON_WINDOWS
5167         (void)unlink(zfilename); /* windows does not replace file with rename() */
5168 #endif
5169         if(rename(tmpfile, zfilename) < 0) {
5170                 log_err("could not rename(%s, %s): %s", tmpfile, zfilename,
5171                         strerror(errno));
5172                 unlink(tmpfile);
5173                 lock_rw_unlock(&z->lock);
5174                 return;
5175         }
5176         lock_rw_unlock(&z->lock);
5177 }
5178
5179 /** reacquire locks and structures. Starts with no locks, ends
5180  * with xfr and z locks, if fail, no z lock */
5181 static int xfr_process_reacquire_locks(struct auth_xfer* xfr,
5182         struct module_env* env, struct auth_zone** z)
5183 {
5184         /* release xfr lock, then, while holding az->lock grab both
5185          * z->lock and xfr->lock */
5186         lock_rw_rdlock(&env->auth_zones->lock);
5187         *z = auth_zone_find(env->auth_zones, xfr->name, xfr->namelen,
5188                 xfr->dclass);
5189         if(!*z) {
5190                 lock_rw_unlock(&env->auth_zones->lock);
5191                 lock_basic_lock(&xfr->lock);
5192                 *z = NULL;
5193                 return 0;
5194         }
5195         lock_rw_wrlock(&(*z)->lock);
5196         lock_basic_lock(&xfr->lock);
5197         lock_rw_unlock(&env->auth_zones->lock);
5198         return 1;
5199 }
5200
5201 /** process chunk list and update zone in memory,
5202  * return false if it did not work */
5203 static int
5204 xfr_process_chunk_list(struct auth_xfer* xfr, struct module_env* env,
5205         int* ixfr_fail)
5206 {
5207         struct auth_zone* z;
5208
5209         /* obtain locks and structures */
5210         lock_basic_unlock(&xfr->lock);
5211         if(!xfr_process_reacquire_locks(xfr, env, &z)) {
5212                 /* the zone is gone, ignore xfr results */
5213                 return 0;
5214         }
5215         /* holding xfr and z locks */
5216
5217         /* apply data */
5218         if(xfr->task_transfer->master->http) {
5219                 if(!apply_http(xfr, z, env->scratch_buffer)) {
5220                         lock_rw_unlock(&z->lock);
5221                         verbose(VERB_ALGO, "http from %s: could not store data",
5222                                 xfr->task_transfer->master->host);
5223                         return 0;
5224                 }
5225         } else if(xfr->task_transfer->on_ixfr &&
5226                 !xfr->task_transfer->on_ixfr_is_axfr) {
5227                 if(!apply_ixfr(xfr, z, env->scratch_buffer)) {
5228                         lock_rw_unlock(&z->lock);
5229                         verbose(VERB_ALGO, "xfr from %s: could not store IXFR"
5230                                 " data", xfr->task_transfer->master->host);
5231                         *ixfr_fail = 1;
5232                         return 0;
5233                 }
5234         } else {
5235                 if(!apply_axfr(xfr, z, env->scratch_buffer)) {
5236                         lock_rw_unlock(&z->lock);
5237                         verbose(VERB_ALGO, "xfr from %s: could not store AXFR"
5238                                 " data", xfr->task_transfer->master->host);
5239                         return 0;
5240                 }
5241         }
5242         xfr->zone_expired = 0;
5243         z->zone_expired = 0;
5244         if(!xfr_find_soa(z, xfr)) {
5245                 lock_rw_unlock(&z->lock);
5246                 verbose(VERB_ALGO, "xfr from %s: no SOA in zone after update"
5247                         " (or malformed RR)", xfr->task_transfer->master->host);
5248                 return 0;
5249         }
5250
5251         /* release xfr lock while verifying zonemd because it may have
5252          * to spawn lookups in the state machines */
5253         lock_basic_unlock(&xfr->lock);
5254         /* holding z lock */
5255         auth_zone_verify_zonemd(z, env, &env->mesh->mods, NULL, 0, 0);
5256         if(z->zone_expired) {
5257                 char zname[256];
5258                 /* ZONEMD must have failed */
5259                 /* reacquire locks, so we hold xfr lock on exit of routine,
5260                  * and both xfr and z again after releasing xfr for potential
5261                  * state machine mesh callbacks */
5262                 lock_rw_unlock(&z->lock);
5263                 if(!xfr_process_reacquire_locks(xfr, env, &z))
5264                         return 0;
5265                 dname_str(xfr->name, zname);
5266                 verbose(VERB_ALGO, "xfr from %s: ZONEMD failed for %s, transfer is failed", xfr->task_transfer->master->host, zname);
5267                 xfr->zone_expired = 1;
5268                 lock_rw_unlock(&z->lock);
5269                 return 0;
5270         }
5271         /* reacquire locks, so we hold xfr lock on exit of routine,
5272          * and both xfr and z again after releasing xfr for potential
5273          * state machine mesh callbacks */
5274         lock_rw_unlock(&z->lock);
5275         if(!xfr_process_reacquire_locks(xfr, env, &z))
5276                 return 0;
5277         /* holding xfr and z locks */
5278
5279         if(xfr->have_zone)
5280                 xfr->lease_time = *env->now;
5281
5282         if(z->rpz)
5283                 rpz_finish_config(z->rpz);
5284
5285         /* unlock */
5286         lock_rw_unlock(&z->lock);
5287
5288         if(verbosity >= VERB_QUERY && xfr->have_zone) {
5289                 char zname[256];
5290                 dname_str(xfr->name, zname);
5291                 verbose(VERB_QUERY, "auth zone %s updated to serial %u", zname,
5292                         (unsigned)xfr->serial);
5293         }
5294         /* see if we need to write to a zonefile */
5295         xfr_write_after_update(xfr, env);
5296         return 1;
5297 }
5298
5299 /** disown task_transfer.  caller must hold xfr.lock */
5300 static void
5301 xfr_transfer_disown(struct auth_xfer* xfr)
5302 {
5303         /* remove timer (from this worker's event base) */
5304         comm_timer_delete(xfr->task_transfer->timer);
5305         xfr->task_transfer->timer = NULL;
5306         /* remove the commpoint */
5307         comm_point_delete(xfr->task_transfer->cp);
5308         xfr->task_transfer->cp = NULL;
5309         /* we don't own this item anymore */
5310         xfr->task_transfer->worker = NULL;
5311         xfr->task_transfer->env = NULL;
5312 }
5313
5314 /** lookup a host name for its addresses, if needed */
5315 static int
5316 xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
5317 {
5318         struct sockaddr_storage addr;
5319         socklen_t addrlen = 0;
5320         struct auth_master* master = xfr->task_transfer->lookup_target;
5321         struct query_info qinfo;
5322         uint16_t qflags = BIT_RD;
5323         uint8_t dname[LDNS_MAX_DOMAINLEN+1];
5324         struct edns_data edns;
5325         sldns_buffer* buf = env->scratch_buffer;
5326         if(!master) return 0;
5327         if(extstrtoaddr(master->host, &addr, &addrlen)) {
5328                 /* not needed, host is in IP addr format */
5329                 return 0;
5330         }
5331         if(master->allow_notify)
5332                 return 0; /* allow-notifies are not transferred from, no
5333                 lookup is needed */
5334
5335         /* use mesh_new_callback to probe for non-addr hosts,
5336          * and then wait for them to be looked up (in cache, or query) */
5337         qinfo.qname_len = sizeof(dname);
5338         if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
5339                 != 0) {
5340                 log_err("cannot parse host name of master %s", master->host);
5341                 return 0;
5342         }
5343         qinfo.qname = dname;
5344         qinfo.qclass = xfr->dclass;
5345         qinfo.qtype = LDNS_RR_TYPE_A;
5346         if(xfr->task_transfer->lookup_aaaa)
5347                 qinfo.qtype = LDNS_RR_TYPE_AAAA;
5348         qinfo.local_alias = NULL;
5349         if(verbosity >= VERB_ALGO) {
5350                 char buf1[512];
5351                 char buf2[LDNS_MAX_DOMAINLEN+1];
5352                 dname_str(xfr->name, buf2);
5353                 snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
5354                         " for task_transfer", buf2);
5355                 log_query_info(VERB_ALGO, buf1, &qinfo);
5356         }
5357         edns.edns_present = 1;
5358         edns.ext_rcode = 0;
5359         edns.edns_version = 0;
5360         edns.bits = EDNS_DO;
5361         edns.opt_list_in = NULL;
5362         edns.opt_list_out = NULL;
5363         edns.opt_list_inplace_cb_out = NULL;
5364         edns.padding_block_size = 0;
5365         if(sldns_buffer_capacity(buf) < 65535)
5366                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
5367         else    edns.udp_size = 65535;
5368
5369         /* unlock xfr during mesh_new_callback() because the callback can be
5370          * called straight away */
5371         lock_basic_unlock(&xfr->lock);
5372         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
5373                 &auth_xfer_transfer_lookup_callback, xfr)) {
5374                 lock_basic_lock(&xfr->lock);
5375                 log_err("out of memory lookup up master %s", master->host);
5376                 return 0;
5377         }
5378         lock_basic_lock(&xfr->lock);
5379         return 1;
5380 }
5381
5382 /** initiate TCP to the target and fetch zone.
5383  * returns true if that was successfully started, and timeout setup. */
5384 static int
5385 xfr_transfer_init_fetch(struct auth_xfer* xfr, struct module_env* env)
5386 {
5387         struct sockaddr_storage addr;
5388         socklen_t addrlen = 0;
5389         struct auth_master* master = xfr->task_transfer->master;
5390         char *auth_name = NULL;
5391         struct timeval t;
5392         int timeout;
5393         if(!master) return 0;
5394         if(master->allow_notify) return 0; /* only for notify */
5395
5396         /* get master addr */
5397         if(xfr->task_transfer->scan_addr) {
5398                 addrlen = xfr->task_transfer->scan_addr->addrlen;
5399                 memmove(&addr, &xfr->task_transfer->scan_addr->addr, addrlen);
5400         } else {
5401                 if(!authextstrtoaddr(master->host, &addr, &addrlen, &auth_name)) {
5402                         /* the ones that are not in addr format are supposed
5403                          * to be looked up.  The lookup has failed however,
5404                          * so skip them */
5405                         char zname[255+1];
5406                         dname_str(xfr->name, zname);
5407                         log_err("%s: failed lookup, cannot transfer from master %s",
5408                                 zname, master->host);
5409                         return 0;
5410                 }
5411         }
5412
5413         /* remove previous TCP connection (if any) */
5414         if(xfr->task_transfer->cp) {
5415                 comm_point_delete(xfr->task_transfer->cp);
5416                 xfr->task_transfer->cp = NULL;
5417         }
5418         if(!xfr->task_transfer->timer) {
5419                 xfr->task_transfer->timer = comm_timer_create(env->worker_base,
5420                         auth_xfer_transfer_timer_callback, xfr);
5421                 if(!xfr->task_transfer->timer) {
5422                         log_err("malloc failure");
5423                         return 0;
5424                 }
5425         }
5426         timeout = AUTH_TRANSFER_TIMEOUT;
5427 #ifndef S_SPLINT_S
5428         t.tv_sec = timeout/1000;
5429         t.tv_usec = (timeout%1000)*1000;
5430 #endif
5431
5432         if(master->http) {
5433                 /* perform http fetch */
5434                 /* store http port number into sockaddr,
5435                  * unless someone used unbound's host@port notation */
5436                 xfr->task_transfer->on_ixfr = 0;
5437                 if(strchr(master->host, '@') == NULL)
5438                         sockaddr_store_port(&addr, addrlen, master->port);
5439                 xfr->task_transfer->cp = outnet_comm_point_for_http(
5440                         env->outnet, auth_xfer_transfer_http_callback, xfr,
5441                         &addr, addrlen, -1, master->ssl, master->host,
5442                         master->file, env->cfg);
5443                 if(!xfr->task_transfer->cp) {
5444                         char zname[255+1], as[256];
5445                         dname_str(xfr->name, zname);
5446                         addr_to_str(&addr, addrlen, as, sizeof(as));
5447                         verbose(VERB_ALGO, "cannot create http cp "
5448                                 "connection for %s to %s", zname, as);
5449                         return 0;
5450                 }
5451                 comm_timer_set(xfr->task_transfer->timer, &t);
5452                 if(verbosity >= VERB_ALGO) {
5453                         char zname[255+1], as[256];
5454                         dname_str(xfr->name, zname);
5455                         addr_to_str(&addr, addrlen, as, sizeof(as));
5456                         verbose(VERB_ALGO, "auth zone %s transfer next HTTP fetch from %s started", zname, as);
5457                 }
5458                 return 1;
5459         }
5460
5461         /* perform AXFR/IXFR */
5462         /* set the packet to be written */
5463         /* create new ID */
5464         xfr->task_transfer->id = GET_RANDOM_ID(env->rnd);
5465         xfr_create_ixfr_packet(xfr, env->scratch_buffer,
5466                 xfr->task_transfer->id, master);
5467
5468         /* connect on fd */
5469         xfr->task_transfer->cp = outnet_comm_point_for_tcp(env->outnet,
5470                 auth_xfer_transfer_tcp_callback, xfr, &addr, addrlen,
5471                 env->scratch_buffer, -1,
5472                 auth_name != NULL, auth_name);
5473         if(!xfr->task_transfer->cp) {
5474                 char zname[255+1], as[256];
5475                 dname_str(xfr->name, zname);
5476                 addr_to_str(&addr, addrlen, as, sizeof(as));
5477                 verbose(VERB_ALGO, "cannot create tcp cp connection for "
5478                         "xfr %s to %s", zname, as);
5479                 return 0;
5480         }
5481         comm_timer_set(xfr->task_transfer->timer, &t);
5482         if(verbosity >= VERB_ALGO) {
5483                 char zname[255+1], as[256];
5484                 dname_str(xfr->name, zname);
5485                 addr_to_str(&addr, addrlen, as, sizeof(as));
5486                 verbose(VERB_ALGO, "auth zone %s transfer next %s fetch from %s started", zname,
5487                         (xfr->task_transfer->on_ixfr?"IXFR":"AXFR"), as);
5488         }
5489         return 1;
5490 }
5491
5492 /** perform next lookup, next transfer TCP, or end and resume wait time task */
5493 static void
5494 xfr_transfer_nexttarget_or_end(struct auth_xfer* xfr, struct module_env* env)
5495 {
5496         log_assert(xfr->task_transfer->worker == env->worker);
5497
5498         /* are we performing lookups? */
5499         while(xfr->task_transfer->lookup_target) {
5500                 if(xfr_transfer_lookup_host(xfr, env)) {
5501                         /* wait for lookup to finish,
5502                          * note that the hostname may be in unbound's cache
5503                          * and we may then get an instant cache response,
5504                          * and that calls the callback just like a full
5505                          * lookup and lookup failures also call callback */
5506                         if(verbosity >= VERB_ALGO) {
5507                                 char zname[255+1];
5508                                 dname_str(xfr->name, zname);
5509                                 verbose(VERB_ALGO, "auth zone %s transfer next target lookup", zname);
5510                         }
5511                         lock_basic_unlock(&xfr->lock);
5512                         return;
5513                 }
5514                 xfr_transfer_move_to_next_lookup(xfr, env);
5515         }
5516
5517         /* initiate TCP and fetch the zone from the master */
5518         /* and set timeout on it */
5519         while(!xfr_transfer_end_of_list(xfr)) {
5520                 xfr->task_transfer->master = xfr_transfer_current_master(xfr);
5521                 if(xfr_transfer_init_fetch(xfr, env)) {
5522                         /* successfully started, wait for callback */
5523                         lock_basic_unlock(&xfr->lock);
5524                         return;
5525                 }
5526                 /* failed to fetch, next master */
5527                 xfr_transfer_nextmaster(xfr);
5528         }
5529         if(verbosity >= VERB_ALGO) {
5530                 char zname[255+1];
5531                 dname_str(xfr->name, zname);
5532                 verbose(VERB_ALGO, "auth zone %s transfer failed, wait", zname);
5533         }
5534
5535         /* we failed to fetch the zone, move to wait task
5536          * use the shorter retry timeout */
5537         xfr_transfer_disown(xfr);
5538
5539         /* pick up the nextprobe task and wait */
5540         if(xfr->task_nextprobe->worker == NULL)
5541                 xfr_set_timeout(xfr, env, 1, 0);
5542         lock_basic_unlock(&xfr->lock);
5543 }
5544
5545 /** add addrs from A or AAAA rrset to the master */
5546 static void
5547 xfr_master_add_addrs(struct auth_master* m, struct ub_packed_rrset_key* rrset,
5548         uint16_t rrtype)
5549 {
5550         size_t i;
5551         struct packed_rrset_data* data;
5552         if(!m || !rrset) return;
5553         if(rrtype != LDNS_RR_TYPE_A && rrtype != LDNS_RR_TYPE_AAAA)
5554                 return;
5555         data = (struct packed_rrset_data*)rrset->entry.data;
5556         for(i=0; i<data->count; i++) {
5557                 struct auth_addr* a;
5558                 size_t len = data->rr_len[i] - 2;
5559                 uint8_t* rdata = data->rr_data[i]+2;
5560                 if(rrtype == LDNS_RR_TYPE_A && len != INET_SIZE)
5561                         continue; /* wrong length for A */
5562                 if(rrtype == LDNS_RR_TYPE_AAAA && len != INET6_SIZE)
5563                         continue; /* wrong length for AAAA */
5564
5565                 /* add and alloc it */
5566                 a = (struct auth_addr*)calloc(1, sizeof(*a));
5567                 if(!a) {
5568                         log_err("out of memory");
5569                         return;
5570                 }
5571                 if(rrtype == LDNS_RR_TYPE_A) {
5572                         struct sockaddr_in* sa;
5573                         a->addrlen = (socklen_t)sizeof(*sa);
5574                         sa = (struct sockaddr_in*)&a->addr;
5575                         sa->sin_family = AF_INET;
5576                         sa->sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5577                         memmove(&sa->sin_addr, rdata, INET_SIZE);
5578                 } else {
5579                         struct sockaddr_in6* sa;
5580                         a->addrlen = (socklen_t)sizeof(*sa);
5581                         sa = (struct sockaddr_in6*)&a->addr;
5582                         sa->sin6_family = AF_INET6;
5583                         sa->sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
5584                         memmove(&sa->sin6_addr, rdata, INET6_SIZE);
5585                 }
5586                 if(verbosity >= VERB_ALGO) {
5587                         char s[64];
5588                         addr_to_str(&a->addr, a->addrlen, s, sizeof(s));
5589                         verbose(VERB_ALGO, "auth host %s lookup %s",
5590                                 m->host, s);
5591                 }
5592                 /* append to list */
5593                 a->next = m->list;
5594                 m->list = a;
5595         }
5596 }
5597
5598 /** callback for task_transfer lookup of host name, of A or AAAA */
5599 void auth_xfer_transfer_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
5600         enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
5601         int ATTR_UNUSED(was_ratelimited))
5602 {
5603         struct auth_xfer* xfr = (struct auth_xfer*)arg;
5604         struct module_env* env;
5605         log_assert(xfr->task_transfer);
5606         lock_basic_lock(&xfr->lock);
5607         env = xfr->task_transfer->env;
5608         if(!env || env->outnet->want_to_quit) {
5609                 lock_basic_unlock(&xfr->lock);
5610                 return; /* stop on quit */
5611         }
5612
5613         /* process result */
5614         if(rcode == LDNS_RCODE_NOERROR) {
5615                 uint16_t wanted_qtype = LDNS_RR_TYPE_A;
5616                 struct regional* temp = env->scratch;
5617                 struct query_info rq;
5618                 struct reply_info* rep;
5619                 if(xfr->task_transfer->lookup_aaaa)
5620                         wanted_qtype = LDNS_RR_TYPE_AAAA;
5621                 memset(&rq, 0, sizeof(rq));
5622                 rep = parse_reply_in_temp_region(buf, temp, &rq);
5623                 if(rep && rq.qtype == wanted_qtype &&
5624                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
5625                         /* parsed successfully */
5626                         struct ub_packed_rrset_key* answer =
5627                                 reply_find_answer_rrset(&rq, rep);
5628                         if(answer) {
5629                                 xfr_master_add_addrs(xfr->task_transfer->
5630                                         lookup_target, answer, wanted_qtype);
5631                         } else {
5632                                 if(verbosity >= VERB_ALGO) {
5633                                         char zname[255+1];
5634                                         dname_str(xfr->name, zname);
5635                                         verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has nodata", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5636                                 }
5637                         }
5638                 } else {
5639                         if(verbosity >= VERB_ALGO) {
5640                                 char zname[255+1];
5641                                 dname_str(xfr->name, zname);
5642                                 verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has no answer", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5643                         }
5644                 }
5645                 regional_free_all(temp);
5646         } else {
5647                 if(verbosity >= VERB_ALGO) {
5648                         char zname[255+1];
5649                         dname_str(xfr->name, zname);
5650                         verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup failed", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
5651                 }
5652         }
5653         if(xfr->task_transfer->lookup_target->list &&
5654                 xfr->task_transfer->lookup_target == xfr_transfer_current_master(xfr))
5655                 xfr->task_transfer->scan_addr = xfr->task_transfer->lookup_target->list;
5656
5657         /* move to lookup AAAA after A lookup, move to next hostname lookup,
5658          * or move to fetch the zone, or, if nothing to do, end task_transfer */
5659         xfr_transfer_move_to_next_lookup(xfr, env);
5660         xfr_transfer_nexttarget_or_end(xfr, env);
5661 }
5662
5663 /** check if xfer (AXFR or IXFR) packet is OK.
5664  * return false if we lost connection (SERVFAIL, or unreadable).
5665  * return false if we need to move from IXFR to AXFR, with gonextonfail
5666  *      set to false, so the same master is tried again, but with AXFR.
5667  * return true if fine to link into data.
5668  * return true with transferdone=true when the transfer has ended.
5669  */
5670 static int
5671 check_xfer_packet(sldns_buffer* pkt, struct auth_xfer* xfr,
5672         int* gonextonfail, int* transferdone)
5673 {
5674         uint8_t* wire = sldns_buffer_begin(pkt);
5675         int i;
5676         if(sldns_buffer_limit(pkt) < LDNS_HEADER_SIZE) {
5677                 verbose(VERB_ALGO, "xfr to %s failed, packet too small",
5678                         xfr->task_transfer->master->host);
5679                 return 0;
5680         }
5681         if(!LDNS_QR_WIRE(wire)) {
5682                 verbose(VERB_ALGO, "xfr to %s failed, packet has no QR flag",
5683                         xfr->task_transfer->master->host);
5684                 return 0;
5685         }
5686         if(LDNS_TC_WIRE(wire)) {
5687                 verbose(VERB_ALGO, "xfr to %s failed, packet has TC flag",
5688                         xfr->task_transfer->master->host);
5689                 return 0;
5690         }
5691         /* check ID */
5692         if(LDNS_ID_WIRE(wire) != xfr->task_transfer->id) {
5693                 verbose(VERB_ALGO, "xfr to %s failed, packet wrong ID",
5694                         xfr->task_transfer->master->host);
5695                 return 0;
5696         }
5697         if(LDNS_RCODE_WIRE(wire) != LDNS_RCODE_NOERROR) {
5698                 char rcode[32];
5699                 sldns_wire2str_rcode_buf((int)LDNS_RCODE_WIRE(wire), rcode,
5700                         sizeof(rcode));
5701                 /* if we are doing IXFR, check for fallback */
5702                 if(xfr->task_transfer->on_ixfr) {
5703                         if(LDNS_RCODE_WIRE(wire) == LDNS_RCODE_NOTIMPL ||
5704                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_SERVFAIL ||
5705                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_REFUSED ||
5706                                 LDNS_RCODE_WIRE(wire) == LDNS_RCODE_FORMERR) {
5707                                 verbose(VERB_ALGO, "xfr to %s, fallback "
5708                                         "from IXFR to AXFR (with rcode %s)",
5709                                         xfr->task_transfer->master->host,
5710                                         rcode);
5711                                 xfr->task_transfer->ixfr_fail = 1;
5712                                 *gonextonfail = 0;
5713                                 return 0;
5714                         }
5715                 }
5716                 verbose(VERB_ALGO, "xfr to %s failed, packet with rcode %s",
5717                         xfr->task_transfer->master->host, rcode);
5718                 return 0;
5719         }
5720         if(LDNS_OPCODE_WIRE(wire) != LDNS_PACKET_QUERY) {
5721                 verbose(VERB_ALGO, "xfr to %s failed, packet with bad opcode",
5722                         xfr->task_transfer->master->host);
5723                 return 0;
5724         }
5725         if(LDNS_QDCOUNT(wire) > 1) {
5726                 verbose(VERB_ALGO, "xfr to %s failed, packet has qdcount %d",
5727                         xfr->task_transfer->master->host,
5728                         (int)LDNS_QDCOUNT(wire));
5729                 return 0;
5730         }
5731
5732         /* check qname */
5733         sldns_buffer_set_position(pkt, LDNS_HEADER_SIZE);
5734         for(i=0; i<(int)LDNS_QDCOUNT(wire); i++) {
5735                 size_t pos = sldns_buffer_position(pkt);
5736                 uint16_t qtype, qclass;
5737                 if(pkt_dname_len(pkt) == 0) {
5738                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5739                                 "malformed dname",
5740                                 xfr->task_transfer->master->host);
5741                         return 0;
5742                 }
5743                 if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5744                         xfr->name) != 0) {
5745                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5746                                 "wrong qname",
5747                                 xfr->task_transfer->master->host);
5748                         return 0;
5749                 }
5750                 if(sldns_buffer_remaining(pkt) < 4) {
5751                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5752                                 "truncated query RR",
5753                                 xfr->task_transfer->master->host);
5754                         return 0;
5755                 }
5756                 qtype = sldns_buffer_read_u16(pkt);
5757                 qclass = sldns_buffer_read_u16(pkt);
5758                 if(qclass != xfr->dclass) {
5759                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5760                                 "wrong qclass",
5761                                 xfr->task_transfer->master->host);
5762                         return 0;
5763                 }
5764                 if(xfr->task_transfer->on_ixfr) {
5765                         if(qtype != LDNS_RR_TYPE_IXFR) {
5766                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5767                                         "with wrong qtype, expected IXFR",
5768                                 xfr->task_transfer->master->host);
5769                                 return 0;
5770                         }
5771                 } else {
5772                         if(qtype != LDNS_RR_TYPE_AXFR) {
5773                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5774                                         "with wrong qtype, expected AXFR",
5775                                 xfr->task_transfer->master->host);
5776                                 return 0;
5777                         }
5778                 }
5779         }
5780
5781         /* check parse of RRs in packet, store first SOA serial
5782          * to be able to detect last SOA (with that serial) to see if done */
5783         /* also check for IXFR 'zone up to date' reply */
5784         for(i=0; i<(int)LDNS_ANCOUNT(wire); i++) {
5785                 size_t pos = sldns_buffer_position(pkt);
5786                 uint16_t tp, rdlen;
5787                 if(pkt_dname_len(pkt) == 0) {
5788                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5789                                 "malformed dname in answer section",
5790                                 xfr->task_transfer->master->host);
5791                         return 0;
5792                 }
5793                 if(sldns_buffer_remaining(pkt) < 10) {
5794                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5795                                 "truncated RR",
5796                                 xfr->task_transfer->master->host);
5797                         return 0;
5798                 }
5799                 tp = sldns_buffer_read_u16(pkt);
5800                 (void)sldns_buffer_read_u16(pkt); /* class */
5801                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5802                 rdlen = sldns_buffer_read_u16(pkt);
5803                 if(sldns_buffer_remaining(pkt) < rdlen) {
5804                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5805                                 "truncated RR rdata",
5806                                 xfr->task_transfer->master->host);
5807                         return 0;
5808                 }
5809
5810                 /* RR parses (haven't checked rdata itself), now look at
5811                  * SOA records to see serial number */
5812                 if(xfr->task_transfer->rr_scan_num == 0 &&
5813                         tp != LDNS_RR_TYPE_SOA) {
5814                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5815                                 "malformed zone transfer, no start SOA",
5816                                 xfr->task_transfer->master->host);
5817                         return 0;
5818                 }
5819                 if(xfr->task_transfer->rr_scan_num == 1 &&
5820                         tp != LDNS_RR_TYPE_SOA) {
5821                         /* second RR is not a SOA record, this is not an IXFR
5822                          * the master is replying with an AXFR */
5823                         xfr->task_transfer->on_ixfr_is_axfr = 1;
5824                 }
5825                 if(tp == LDNS_RR_TYPE_SOA) {
5826                         uint32_t serial;
5827                         if(rdlen < 22) {
5828                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5829                                         "with SOA with malformed rdata",
5830                                         xfr->task_transfer->master->host);
5831                                 return 0;
5832                         }
5833                         if(dname_pkt_compare(pkt, sldns_buffer_at(pkt, pos),
5834                                 xfr->name) != 0) {
5835                                 verbose(VERB_ALGO, "xfr to %s failed, packet "
5836                                         "with SOA with wrong dname",
5837                                         xfr->task_transfer->master->host);
5838                                 return 0;
5839                         }
5840
5841                         /* read serial number of SOA */
5842                         serial = sldns_buffer_read_u32_at(pkt,
5843                                 sldns_buffer_position(pkt)+rdlen-20);
5844
5845                         /* check for IXFR 'zone has SOA x' reply */
5846                         if(xfr->task_transfer->on_ixfr &&
5847                                 xfr->task_transfer->rr_scan_num == 0 &&
5848                                 LDNS_ANCOUNT(wire)==1) {
5849                                 verbose(VERB_ALGO, "xfr to %s ended, "
5850                                         "IXFR reply that zone has serial %u,"
5851                                         " fallback from IXFR to AXFR",
5852                                         xfr->task_transfer->master->host,
5853                                         (unsigned)serial);
5854                                 xfr->task_transfer->ixfr_fail = 1;
5855                                 *gonextonfail = 0;
5856                                 return 0;
5857                         }
5858
5859                         /* if first SOA, store serial number */
5860                         if(xfr->task_transfer->got_xfr_serial == 0) {
5861                                 xfr->task_transfer->got_xfr_serial = 1;
5862                                 xfr->task_transfer->incoming_xfr_serial =
5863                                         serial;
5864                                 verbose(VERB_ALGO, "xfr %s: contains "
5865                                         "SOA serial %u",
5866                                         xfr->task_transfer->master->host,
5867                                         (unsigned)serial);
5868                         /* see if end of AXFR */
5869                         } else if(!xfr->task_transfer->on_ixfr ||
5870                                 xfr->task_transfer->on_ixfr_is_axfr) {
5871                                 /* second SOA with serial is the end
5872                                  * for AXFR */
5873                                 *transferdone = 1;
5874                                 verbose(VERB_ALGO, "xfr %s: last AXFR packet",
5875                                         xfr->task_transfer->master->host);
5876                         /* for IXFR, count SOA records with that serial */
5877                         } else if(xfr->task_transfer->incoming_xfr_serial ==
5878                                 serial && xfr->task_transfer->got_xfr_serial
5879                                 == 1) {
5880                                 xfr->task_transfer->got_xfr_serial++;
5881                         /* if not first soa, if serial==firstserial, the
5882                          * third time we are at the end, for IXFR */
5883                         } else if(xfr->task_transfer->incoming_xfr_serial ==
5884                                 serial && xfr->task_transfer->got_xfr_serial
5885                                 == 2) {
5886                                 verbose(VERB_ALGO, "xfr %s: last IXFR packet",
5887                                         xfr->task_transfer->master->host);
5888                                 *transferdone = 1;
5889                                 /* continue parse check, if that succeeds,
5890                                  * transfer is done */
5891                         }
5892                 }
5893                 xfr->task_transfer->rr_scan_num++;
5894
5895                 /* skip over RR rdata to go to the next RR */
5896                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5897         }
5898
5899         /* check authority section */
5900         /* we skip over the RRs checking packet format */
5901         for(i=0; i<(int)LDNS_NSCOUNT(wire); i++) {
5902                 uint16_t rdlen;
5903                 if(pkt_dname_len(pkt) == 0) {
5904                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5905                                 "malformed dname in authority section",
5906                                 xfr->task_transfer->master->host);
5907                         return 0;
5908                 }
5909                 if(sldns_buffer_remaining(pkt) < 10) {
5910                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5911                                 "truncated RR",
5912                                 xfr->task_transfer->master->host);
5913                         return 0;
5914                 }
5915                 (void)sldns_buffer_read_u16(pkt); /* type */
5916                 (void)sldns_buffer_read_u16(pkt); /* class */
5917                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5918                 rdlen = sldns_buffer_read_u16(pkt);
5919                 if(sldns_buffer_remaining(pkt) < rdlen) {
5920                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5921                                 "truncated RR rdata",
5922                                 xfr->task_transfer->master->host);
5923                         return 0;
5924                 }
5925                 /* skip over RR rdata to go to the next RR */
5926                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5927         }
5928
5929         /* check additional section */
5930         for(i=0; i<(int)LDNS_ARCOUNT(wire); i++) {
5931                 uint16_t rdlen;
5932                 if(pkt_dname_len(pkt) == 0) {
5933                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5934                                 "malformed dname in additional section",
5935                                 xfr->task_transfer->master->host);
5936                         return 0;
5937                 }
5938                 if(sldns_buffer_remaining(pkt) < 10) {
5939                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5940                                 "truncated RR",
5941                                 xfr->task_transfer->master->host);
5942                         return 0;
5943                 }
5944                 (void)sldns_buffer_read_u16(pkt); /* type */
5945                 (void)sldns_buffer_read_u16(pkt); /* class */
5946                 (void)sldns_buffer_read_u32(pkt); /* ttl */
5947                 rdlen = sldns_buffer_read_u16(pkt);
5948                 if(sldns_buffer_remaining(pkt) < rdlen) {
5949                         verbose(VERB_ALGO, "xfr to %s failed, packet with "
5950                                 "truncated RR rdata",
5951                                 xfr->task_transfer->master->host);
5952                         return 0;
5953                 }
5954                 /* skip over RR rdata to go to the next RR */
5955                 sldns_buffer_skip(pkt, (ssize_t)rdlen);
5956         }
5957
5958         return 1;
5959 }
5960
5961 /** Link the data from this packet into the worklist of transferred data */
5962 static int
5963 xfer_link_data(sldns_buffer* pkt, struct auth_xfer* xfr)
5964 {
5965         /* alloc it */
5966         struct auth_chunk* e;
5967         e = (struct auth_chunk*)calloc(1, sizeof(*e));
5968         if(!e) return 0;
5969         e->next = NULL;
5970         e->len = sldns_buffer_limit(pkt);
5971         e->data = memdup(sldns_buffer_begin(pkt), e->len);
5972         if(!e->data) {
5973                 free(e);
5974                 return 0;
5975         }
5976
5977         /* alloc succeeded, link into list */
5978         if(!xfr->task_transfer->chunks_first)
5979                 xfr->task_transfer->chunks_first = e;
5980         if(xfr->task_transfer->chunks_last)
5981                 xfr->task_transfer->chunks_last->next = e;
5982         xfr->task_transfer->chunks_last = e;
5983         return 1;
5984 }
5985
5986 /** task transfer.  the list of data is complete. process it and if failed
5987  * move to next master, if succeeded, end the task transfer */
5988 static void
5989 process_list_end_transfer(struct auth_xfer* xfr, struct module_env* env)
5990 {
5991         int ixfr_fail = 0;
5992         if(xfr_process_chunk_list(xfr, env, &ixfr_fail)) {
5993                 /* it worked! */
5994                 auth_chunks_delete(xfr->task_transfer);
5995
5996                 /* we fetched the zone, move to wait task */
5997                 xfr_transfer_disown(xfr);
5998
5999                 if(xfr->notify_received && (!xfr->notify_has_serial ||
6000                         (xfr->notify_has_serial &&
6001                         xfr_serial_means_update(xfr, xfr->notify_serial)))) {
6002                         uint32_t sr = xfr->notify_serial;
6003                         int has_sr = xfr->notify_has_serial;
6004                         /* we received a notify while probe/transfer was
6005                          * in progress.  start a new probe and transfer */
6006                         xfr->notify_received = 0;
6007                         xfr->notify_has_serial = 0;
6008                         xfr->notify_serial = 0;
6009                         if(!xfr_start_probe(xfr, env, NULL)) {
6010                                 /* if we couldn't start it, already in
6011                                  * progress; restore notify serial,
6012                                  * while xfr still locked */
6013                                 xfr->notify_received = 1;
6014                                 xfr->notify_has_serial = has_sr;
6015                                 xfr->notify_serial = sr;
6016                                 lock_basic_unlock(&xfr->lock);
6017                         }
6018                         return;
6019                 } else {
6020                         /* pick up the nextprobe task and wait (normail wait time) */
6021                         if(xfr->task_nextprobe->worker == NULL)
6022                                 xfr_set_timeout(xfr, env, 0, 0);
6023                 }
6024                 lock_basic_unlock(&xfr->lock);
6025                 return;
6026         }
6027         /* processing failed */
6028         /* when done, delete data from list */
6029         auth_chunks_delete(xfr->task_transfer);
6030         if(ixfr_fail) {
6031                 xfr->task_transfer->ixfr_fail = 1;
6032         } else {
6033                 xfr_transfer_nextmaster(xfr);
6034         }
6035         xfr_transfer_nexttarget_or_end(xfr, env);
6036 }
6037
6038 /** callback for the task_transfer timer */
6039 void
6040 auth_xfer_transfer_timer_callback(void* arg)
6041 {
6042         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6043         struct module_env* env;
6044         int gonextonfail = 1;
6045         log_assert(xfr->task_transfer);
6046         lock_basic_lock(&xfr->lock);
6047         env = xfr->task_transfer->env;
6048         if(!env || env->outnet->want_to_quit) {
6049                 lock_basic_unlock(&xfr->lock);
6050                 return; /* stop on quit */
6051         }
6052
6053         verbose(VERB_ALGO, "xfr stopped, connection timeout to %s",
6054                 xfr->task_transfer->master->host);
6055
6056         /* see if IXFR caused the failure, if so, try AXFR */
6057         if(xfr->task_transfer->on_ixfr) {
6058                 xfr->task_transfer->ixfr_possible_timeout_count++;
6059                 if(xfr->task_transfer->ixfr_possible_timeout_count >=
6060                         NUM_TIMEOUTS_FALLBACK_IXFR) {
6061                         verbose(VERB_ALGO, "xfr to %s, fallback "
6062                                 "from IXFR to AXFR (because of timeouts)",
6063                                 xfr->task_transfer->master->host);
6064                         xfr->task_transfer->ixfr_fail = 1;
6065                         gonextonfail = 0;
6066                 }
6067         }
6068
6069         /* delete transferred data from list */
6070         auth_chunks_delete(xfr->task_transfer);
6071         comm_point_delete(xfr->task_transfer->cp);
6072         xfr->task_transfer->cp = NULL;
6073         if(gonextonfail)
6074                 xfr_transfer_nextmaster(xfr);
6075         xfr_transfer_nexttarget_or_end(xfr, env);
6076 }
6077
6078 /** callback for task_transfer tcp connections */
6079 int
6080 auth_xfer_transfer_tcp_callback(struct comm_point* c, void* arg, int err,
6081         struct comm_reply* ATTR_UNUSED(repinfo))
6082 {
6083         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6084         struct module_env* env;
6085         int gonextonfail = 1;
6086         int transferdone = 0;
6087         log_assert(xfr->task_transfer);
6088         lock_basic_lock(&xfr->lock);
6089         env = xfr->task_transfer->env;
6090         if(!env || env->outnet->want_to_quit) {
6091                 lock_basic_unlock(&xfr->lock);
6092                 return 0; /* stop on quit */
6093         }
6094         /* stop the timer */
6095         comm_timer_disable(xfr->task_transfer->timer);
6096
6097         if(err != NETEVENT_NOERROR) {
6098                 /* connection failed, closed, or timeout */
6099                 /* stop this transfer, cleanup
6100                  * and continue task_transfer*/
6101                 verbose(VERB_ALGO, "xfr stopped, connection lost to %s",
6102                         xfr->task_transfer->master->host);
6103
6104                 /* see if IXFR caused the failure, if so, try AXFR */
6105                 if(xfr->task_transfer->on_ixfr) {
6106                         xfr->task_transfer->ixfr_possible_timeout_count++;
6107                         if(xfr->task_transfer->ixfr_possible_timeout_count >=
6108                                 NUM_TIMEOUTS_FALLBACK_IXFR) {
6109                                 verbose(VERB_ALGO, "xfr to %s, fallback "
6110                                         "from IXFR to AXFR (because of timeouts)",
6111                                         xfr->task_transfer->master->host);
6112                                 xfr->task_transfer->ixfr_fail = 1;
6113                                 gonextonfail = 0;
6114                         }
6115                 }
6116
6117         failed:
6118                 /* delete transferred data from list */
6119                 auth_chunks_delete(xfr->task_transfer);
6120                 comm_point_delete(xfr->task_transfer->cp);
6121                 xfr->task_transfer->cp = NULL;
6122                 if(gonextonfail)
6123                         xfr_transfer_nextmaster(xfr);
6124                 xfr_transfer_nexttarget_or_end(xfr, env);
6125                 return 0;
6126         }
6127         /* note that IXFR worked without timeout */
6128         if(xfr->task_transfer->on_ixfr)
6129                 xfr->task_transfer->ixfr_possible_timeout_count = 0;
6130
6131         /* handle returned packet */
6132         /* if it fails, cleanup and end this transfer */
6133         /* if it needs to fallback from IXFR to AXFR, do that */
6134         if(!check_xfer_packet(c->buffer, xfr, &gonextonfail, &transferdone)) {
6135                 goto failed;
6136         }
6137         /* if it is good, link it into the list of data */
6138         /* if the link into list of data fails (malloc fail) cleanup and end */
6139         if(!xfer_link_data(c->buffer, xfr)) {
6140                 verbose(VERB_ALGO, "xfr stopped to %s, malloc failed",
6141                         xfr->task_transfer->master->host);
6142                 goto failed;
6143         }
6144         /* if the transfer is done now, disconnect and process the list */
6145         if(transferdone) {
6146                 comm_point_delete(xfr->task_transfer->cp);
6147                 xfr->task_transfer->cp = NULL;
6148                 process_list_end_transfer(xfr, env);
6149                 return 0;
6150         }
6151
6152         /* if we want to read more messages, setup the commpoint to read
6153          * a DNS packet, and the timeout */
6154         lock_basic_unlock(&xfr->lock);
6155         c->tcp_is_reading = 1;
6156         sldns_buffer_clear(c->buffer);
6157         comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
6158         return 0;
6159 }
6160
6161 /** callback for task_transfer http connections */
6162 int
6163 auth_xfer_transfer_http_callback(struct comm_point* c, void* arg, int err,
6164         struct comm_reply* repinfo)
6165 {
6166         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6167         struct module_env* env;
6168         log_assert(xfr->task_transfer);
6169         lock_basic_lock(&xfr->lock);
6170         env = xfr->task_transfer->env;
6171         if(!env || env->outnet->want_to_quit) {
6172                 lock_basic_unlock(&xfr->lock);
6173                 return 0; /* stop on quit */
6174         }
6175         verbose(VERB_ALGO, "auth zone transfer http callback");
6176         /* stop the timer */
6177         comm_timer_disable(xfr->task_transfer->timer);
6178
6179         if(err != NETEVENT_NOERROR && err != NETEVENT_DONE) {
6180                 /* connection failed, closed, or timeout */
6181                 /* stop this transfer, cleanup
6182                  * and continue task_transfer*/
6183                 verbose(VERB_ALGO, "http stopped, connection lost to %s",
6184                         xfr->task_transfer->master->host);
6185         failed:
6186                 /* delete transferred data from list */
6187                 auth_chunks_delete(xfr->task_transfer);
6188                 if(repinfo) repinfo->c = NULL; /* signal cp deleted to
6189                                 the routine calling this callback */
6190                 comm_point_delete(xfr->task_transfer->cp);
6191                 xfr->task_transfer->cp = NULL;
6192                 xfr_transfer_nextmaster(xfr);
6193                 xfr_transfer_nexttarget_or_end(xfr, env);
6194                 return 0;
6195         }
6196
6197         /* if it is good, link it into the list of data */
6198         /* if the link into list of data fails (malloc fail) cleanup and end */
6199         if(sldns_buffer_limit(c->buffer) > 0) {
6200                 verbose(VERB_ALGO, "auth zone http queued up %d bytes",
6201                         (int)sldns_buffer_limit(c->buffer));
6202                 if(!xfer_link_data(c->buffer, xfr)) {
6203                         verbose(VERB_ALGO, "http stopped to %s, malloc failed",
6204                                 xfr->task_transfer->master->host);
6205                         goto failed;
6206                 }
6207         }
6208         /* if the transfer is done now, disconnect and process the list */
6209         if(err == NETEVENT_DONE) {
6210                 if(repinfo) repinfo->c = NULL; /* signal cp deleted to
6211                                 the routine calling this callback */
6212                 comm_point_delete(xfr->task_transfer->cp);
6213                 xfr->task_transfer->cp = NULL;
6214                 process_list_end_transfer(xfr, env);
6215                 return 0;
6216         }
6217
6218         /* if we want to read more messages, setup the commpoint to read
6219          * a DNS packet, and the timeout */
6220         lock_basic_unlock(&xfr->lock);
6221         c->tcp_is_reading = 1;
6222         sldns_buffer_clear(c->buffer);
6223         comm_point_start_listening(c, -1, AUTH_TRANSFER_TIMEOUT);
6224         return 0;
6225 }
6226
6227
6228 /** start transfer task by this worker , xfr is locked. */
6229 static void
6230 xfr_start_transfer(struct auth_xfer* xfr, struct module_env* env,
6231         struct auth_master* master)
6232 {
6233         log_assert(xfr->task_transfer != NULL);
6234         log_assert(xfr->task_transfer->worker == NULL);
6235         log_assert(xfr->task_transfer->chunks_first == NULL);
6236         log_assert(xfr->task_transfer->chunks_last == NULL);
6237         xfr->task_transfer->worker = env->worker;
6238         xfr->task_transfer->env = env;
6239
6240         /* init transfer process */
6241         /* find that master in the transfer's list of masters? */
6242         xfr_transfer_start_list(xfr, master);
6243         /* start lookup for hostnames in transfer master list */
6244         xfr_transfer_start_lookups(xfr);
6245
6246         /* initiate TCP, and set timeout on it */
6247         xfr_transfer_nexttarget_or_end(xfr, env);
6248 }
6249
6250 /** disown task_probe.  caller must hold xfr.lock */
6251 static void
6252 xfr_probe_disown(struct auth_xfer* xfr)
6253 {
6254         /* remove timer (from this worker's event base) */
6255         comm_timer_delete(xfr->task_probe->timer);
6256         xfr->task_probe->timer = NULL;
6257         /* remove the commpoint */
6258         comm_point_delete(xfr->task_probe->cp);
6259         xfr->task_probe->cp = NULL;
6260         /* we don't own this item anymore */
6261         xfr->task_probe->worker = NULL;
6262         xfr->task_probe->env = NULL;
6263 }
6264
6265 /** send the UDP probe to the master, this is part of task_probe */
6266 static int
6267 xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
6268         int timeout)
6269 {
6270         struct sockaddr_storage addr;
6271         socklen_t addrlen = 0;
6272         struct timeval t;
6273         /* pick master */
6274         struct auth_master* master = xfr_probe_current_master(xfr);
6275         char *auth_name = NULL;
6276         if(!master) return 0;
6277         if(master->allow_notify) return 0; /* only for notify */
6278         if(master->http) return 0; /* only masters get SOA UDP probe,
6279                 not urls, if those are in this list */
6280
6281         /* get master addr */
6282         if(xfr->task_probe->scan_addr) {
6283                 addrlen = xfr->task_probe->scan_addr->addrlen;
6284                 memmove(&addr, &xfr->task_probe->scan_addr->addr, addrlen);
6285         } else {
6286                 if(!authextstrtoaddr(master->host, &addr, &addrlen, &auth_name)) {
6287                         /* the ones that are not in addr format are supposed
6288                          * to be looked up.  The lookup has failed however,
6289                          * so skip them */
6290                         char zname[255+1];
6291                         dname_str(xfr->name, zname);
6292                         log_err("%s: failed lookup, cannot probe to master %s",
6293                                 zname, master->host);
6294                         return 0;
6295                 }
6296                 if (auth_name != NULL) {
6297                         if (addr.ss_family == AF_INET
6298                         &&  (int)ntohs(((struct sockaddr_in *)&addr)->sin_port)
6299                             == env->cfg->ssl_port)
6300                                 ((struct sockaddr_in *)&addr)->sin_port
6301                                         = htons((uint16_t)env->cfg->port);
6302                         else if (addr.ss_family == AF_INET6
6303                         &&  (int)ntohs(((struct sockaddr_in6 *)&addr)->sin6_port)
6304                             == env->cfg->ssl_port)
6305                                 ((struct sockaddr_in6 *)&addr)->sin6_port
6306                                         = htons((uint16_t)env->cfg->port);
6307                 }
6308         }
6309
6310         /* create packet */
6311         /* create new ID for new probes, but not on timeout retries,
6312          * this means we'll accept replies to previous retries to same ip */
6313         if(timeout == AUTH_PROBE_TIMEOUT)
6314                 xfr->task_probe->id = GET_RANDOM_ID(env->rnd);
6315         xfr_create_soa_probe_packet(xfr, env->scratch_buffer,
6316                 xfr->task_probe->id);
6317         /* we need to remove the cp if we have a different ip4/ip6 type now */
6318         if(xfr->task_probe->cp &&
6319                 ((xfr->task_probe->cp_is_ip6 && !addr_is_ip6(&addr, addrlen)) ||
6320                 (!xfr->task_probe->cp_is_ip6 && addr_is_ip6(&addr, addrlen)))
6321                 ) {
6322                 comm_point_delete(xfr->task_probe->cp);
6323                 xfr->task_probe->cp = NULL;
6324         }
6325         if(!xfr->task_probe->cp) {
6326                 if(addr_is_ip6(&addr, addrlen))
6327                         xfr->task_probe->cp_is_ip6 = 1;
6328                 else    xfr->task_probe->cp_is_ip6 = 0;
6329                 xfr->task_probe->cp = outnet_comm_point_for_udp(env->outnet,
6330                         auth_xfer_probe_udp_callback, xfr, &addr, addrlen);
6331                 if(!xfr->task_probe->cp) {
6332                         char zname[255+1], as[256];
6333                         dname_str(xfr->name, zname);
6334                         addr_to_str(&addr, addrlen, as, sizeof(as));
6335                         verbose(VERB_ALGO, "cannot create udp cp for "
6336                                 "probe %s to %s", zname, as);
6337                         return 0;
6338                 }
6339         }
6340         if(!xfr->task_probe->timer) {
6341                 xfr->task_probe->timer = comm_timer_create(env->worker_base,
6342                         auth_xfer_probe_timer_callback, xfr);
6343                 if(!xfr->task_probe->timer) {
6344                         log_err("malloc failure");
6345                         return 0;
6346                 }
6347         }
6348
6349         /* send udp packet */
6350         if(!comm_point_send_udp_msg(xfr->task_probe->cp, env->scratch_buffer,
6351                 (struct sockaddr*)&addr, addrlen, 0)) {
6352                 char zname[255+1], as[256];
6353                 dname_str(xfr->name, zname);
6354                 addr_to_str(&addr, addrlen, as, sizeof(as));
6355                 verbose(VERB_ALGO, "failed to send soa probe for %s to %s",
6356                         zname, as);
6357                 return 0;
6358         }
6359         if(verbosity >= VERB_ALGO) {
6360                 char zname[255+1], as[256];
6361                 dname_str(xfr->name, zname);
6362                 addr_to_str(&addr, addrlen, as, sizeof(as));
6363                 verbose(VERB_ALGO, "auth zone %s soa probe sent to %s", zname,
6364                         as);
6365         }
6366         xfr->task_probe->timeout = timeout;
6367 #ifndef S_SPLINT_S
6368         t.tv_sec = timeout/1000;
6369         t.tv_usec = (timeout%1000)*1000;
6370 #endif
6371         comm_timer_set(xfr->task_probe->timer, &t);
6372
6373         return 1;
6374 }
6375
6376 /** callback for task_probe timer */
6377 void
6378 auth_xfer_probe_timer_callback(void* arg)
6379 {
6380         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6381         struct module_env* env;
6382         log_assert(xfr->task_probe);
6383         lock_basic_lock(&xfr->lock);
6384         env = xfr->task_probe->env;
6385         if(!env || env->outnet->want_to_quit) {
6386                 lock_basic_unlock(&xfr->lock);
6387                 return; /* stop on quit */
6388         }
6389
6390         if(verbosity >= VERB_ALGO) {
6391                 char zname[255+1];
6392                 dname_str(xfr->name, zname);
6393                 verbose(VERB_ALGO, "auth zone %s soa probe timeout", zname);
6394         }
6395         if(xfr->task_probe->timeout <= AUTH_PROBE_TIMEOUT_STOP) {
6396                 /* try again with bigger timeout */
6397                 if(xfr_probe_send_probe(xfr, env, xfr->task_probe->timeout*2)) {
6398                         lock_basic_unlock(&xfr->lock);
6399                         return;
6400                 }
6401         }
6402         /* delete commpoint so a new one is created, with a fresh port nr */
6403         comm_point_delete(xfr->task_probe->cp);
6404         xfr->task_probe->cp = NULL;
6405
6406         /* too many timeouts (or fail to send), move to next or end */
6407         xfr_probe_nextmaster(xfr);
6408         xfr_probe_send_or_end(xfr, env);
6409 }
6410
6411 /** callback for task_probe udp packets */
6412 int
6413 auth_xfer_probe_udp_callback(struct comm_point* c, void* arg, int err,
6414         struct comm_reply* repinfo)
6415 {
6416         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6417         struct module_env* env;
6418         log_assert(xfr->task_probe);
6419         lock_basic_lock(&xfr->lock);
6420         env = xfr->task_probe->env;
6421         if(!env || env->outnet->want_to_quit) {
6422                 lock_basic_unlock(&xfr->lock);
6423                 return 0; /* stop on quit */
6424         }
6425
6426         /* the comm_point_udp_callback is in a for loop for NUM_UDP_PER_SELECT
6427          * and we set rep.c=NULL to stop if from looking inside the commpoint*/
6428         repinfo->c = NULL;
6429         /* stop the timer */
6430         comm_timer_disable(xfr->task_probe->timer);
6431
6432         /* see if we got a packet and what that means */
6433         if(err == NETEVENT_NOERROR) {
6434                 uint32_t serial = 0;
6435                 if(check_packet_ok(c->buffer, LDNS_RR_TYPE_SOA, xfr,
6436                         &serial)) {
6437                         /* successful lookup */
6438                         if(verbosity >= VERB_ALGO) {
6439                                 char buf[256];
6440                                 dname_str(xfr->name, buf);
6441                                 verbose(VERB_ALGO, "auth zone %s: soa probe "
6442                                         "serial is %u", buf, (unsigned)serial);
6443                         }
6444                         /* see if this serial indicates that the zone has
6445                          * to be updated */
6446                         if(xfr_serial_means_update(xfr, serial)) {
6447                                 /* if updated, start the transfer task, if needed */
6448                                 verbose(VERB_ALGO, "auth_zone updated, start transfer");
6449                                 if(xfr->task_transfer->worker == NULL) {
6450                                         struct auth_master* master =
6451                                                 xfr_probe_current_master(xfr);
6452                                         /* if we have download URLs use them
6453                                          * in preference to this master we
6454                                          * just probed the SOA from */
6455                                         if(xfr->task_transfer->masters &&
6456                                                 xfr->task_transfer->masters->http)
6457                                                 master = NULL;
6458                                         xfr_probe_disown(xfr);
6459                                         xfr_start_transfer(xfr, env, master);
6460                                         return 0;
6461
6462                                 }
6463                                 /* other tasks are running, we don't do this anymore */
6464                                 xfr_probe_disown(xfr);
6465                                 lock_basic_unlock(&xfr->lock);
6466                                 /* return, we don't sent a reply to this udp packet,
6467                                  * and we setup the tasks to do next */
6468                                 return 0;
6469                         } else {
6470                                 verbose(VERB_ALGO, "auth_zone master reports unchanged soa serial");
6471                                 /* we if cannot find updates amongst the
6472                                  * masters, this means we then have a new lease
6473                                  * on the zone */
6474                                 xfr->task_probe->have_new_lease = 1;
6475                         }
6476                 } else {
6477                         if(verbosity >= VERB_ALGO) {
6478                                 char buf[256];
6479                                 dname_str(xfr->name, buf);
6480                                 verbose(VERB_ALGO, "auth zone %s: bad reply to soa probe", buf);
6481                         }
6482                 }
6483         } else {
6484                 if(verbosity >= VERB_ALGO) {
6485                         char buf[256];
6486                         dname_str(xfr->name, buf);
6487                         verbose(VERB_ALGO, "auth zone %s: soa probe failed", buf);
6488                 }
6489         }
6490
6491         /* failed lookup or not an update */
6492         /* delete commpoint so a new one is created, with a fresh port nr */
6493         comm_point_delete(xfr->task_probe->cp);
6494         xfr->task_probe->cp = NULL;
6495
6496         /* if the result was not a successful probe, we need
6497          * to send the next one */
6498         xfr_probe_nextmaster(xfr);
6499         xfr_probe_send_or_end(xfr, env);
6500         return 0;
6501 }
6502
6503 /** lookup a host name for its addresses, if needed */
6504 static int
6505 xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
6506 {
6507         struct sockaddr_storage addr;
6508         socklen_t addrlen = 0;
6509         struct auth_master* master = xfr->task_probe->lookup_target;
6510         struct query_info qinfo;
6511         uint16_t qflags = BIT_RD;
6512         uint8_t dname[LDNS_MAX_DOMAINLEN+1];
6513         struct edns_data edns;
6514         sldns_buffer* buf = env->scratch_buffer;
6515         if(!master) return 0;
6516         if(extstrtoaddr(master->host, &addr, &addrlen)) {
6517                 /* not needed, host is in IP addr format */
6518                 return 0;
6519         }
6520         if(master->allow_notify && !master->http &&
6521                 strchr(master->host, '/') != NULL &&
6522                 strchr(master->host, '/') == strrchr(master->host, '/')) {
6523                 return 0; /* is IP/prefix format, not something to look up */
6524         }
6525
6526         /* use mesh_new_callback to probe for non-addr hosts,
6527          * and then wait for them to be looked up (in cache, or query) */
6528         qinfo.qname_len = sizeof(dname);
6529         if(sldns_str2wire_dname_buf(master->host, dname, &qinfo.qname_len)
6530                 != 0) {
6531                 log_err("cannot parse host name of master %s", master->host);
6532                 return 0;
6533         }
6534         qinfo.qname = dname;
6535         qinfo.qclass = xfr->dclass;
6536         qinfo.qtype = LDNS_RR_TYPE_A;
6537         if(xfr->task_probe->lookup_aaaa)
6538                 qinfo.qtype = LDNS_RR_TYPE_AAAA;
6539         qinfo.local_alias = NULL;
6540         if(verbosity >= VERB_ALGO) {
6541                 char buf1[512];
6542                 char buf2[LDNS_MAX_DOMAINLEN+1];
6543                 dname_str(xfr->name, buf2);
6544                 snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
6545                         " for task_probe", buf2);
6546                 log_query_info(VERB_ALGO, buf1, &qinfo);
6547         }
6548         edns.edns_present = 1;
6549         edns.ext_rcode = 0;
6550         edns.edns_version = 0;
6551         edns.bits = EDNS_DO;
6552         edns.opt_list_in = NULL;
6553         edns.opt_list_out = NULL;
6554         edns.opt_list_inplace_cb_out = NULL;
6555         edns.padding_block_size = 0;
6556         if(sldns_buffer_capacity(buf) < 65535)
6557                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
6558         else    edns.udp_size = 65535;
6559
6560         /* unlock xfr during mesh_new_callback() because the callback can be
6561          * called straight away */
6562         lock_basic_unlock(&xfr->lock);
6563         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
6564                 &auth_xfer_probe_lookup_callback, xfr)) {
6565                 lock_basic_lock(&xfr->lock);
6566                 log_err("out of memory lookup up master %s", master->host);
6567                 return 0;
6568         }
6569         lock_basic_lock(&xfr->lock);
6570         return 1;
6571 }
6572
6573 /** move to sending the probe packets, next if fails. task_probe */
6574 static void
6575 xfr_probe_send_or_end(struct auth_xfer* xfr, struct module_env* env)
6576 {
6577         /* are we doing hostname lookups? */
6578         while(xfr->task_probe->lookup_target) {
6579                 if(xfr_probe_lookup_host(xfr, env)) {
6580                         /* wait for lookup to finish,
6581                          * note that the hostname may be in unbound's cache
6582                          * and we may then get an instant cache response,
6583                          * and that calls the callback just like a full
6584                          * lookup and lookup failures also call callback */
6585                         if(verbosity >= VERB_ALGO) {
6586                                 char zname[255+1];
6587                                 dname_str(xfr->name, zname);
6588                                 verbose(VERB_ALGO, "auth zone %s probe next target lookup", zname);
6589                         }
6590                         lock_basic_unlock(&xfr->lock);
6591                         return;
6592                 }
6593                 xfr_probe_move_to_next_lookup(xfr, env);
6594         }
6595         /* probe of list has ended.  Create or refresh the list of of
6596          * allow_notify addrs */
6597         probe_copy_masters_for_allow_notify(xfr);
6598         if(verbosity >= VERB_ALGO) {
6599                 char zname[255+1];
6600                 dname_str(xfr->name, zname);
6601                 verbose(VERB_ALGO, "auth zone %s probe: notify addrs updated", zname);
6602         }
6603         if(xfr->task_probe->only_lookup) {
6604                 /* only wanted lookups for copy, stop probe and start wait */
6605                 xfr->task_probe->only_lookup = 0;
6606                 if(verbosity >= VERB_ALGO) {
6607                         char zname[255+1];
6608                         dname_str(xfr->name, zname);
6609                         verbose(VERB_ALGO, "auth zone %s probe: finished only_lookup", zname);
6610                 }
6611                 xfr_probe_disown(xfr);
6612                 if(xfr->task_nextprobe->worker == NULL)
6613                         xfr_set_timeout(xfr, env, 0, 0);
6614                 lock_basic_unlock(&xfr->lock);
6615                 return;
6616         }
6617
6618         /* send probe packets */
6619         while(!xfr_probe_end_of_list(xfr)) {
6620                 if(xfr_probe_send_probe(xfr, env, AUTH_PROBE_TIMEOUT)) {
6621                         /* successfully sent probe, wait for callback */
6622                         lock_basic_unlock(&xfr->lock);
6623                         return;
6624                 }
6625                 /* failed to send probe, next master */
6626                 xfr_probe_nextmaster(xfr);
6627         }
6628
6629         /* done with probe sequence, wait */
6630         if(xfr->task_probe->have_new_lease) {
6631                 /* if zone not updated, start the wait timer again */
6632                 if(verbosity >= VERB_ALGO) {
6633                         char zname[255+1];
6634                         dname_str(xfr->name, zname);
6635                         verbose(VERB_ALGO, "auth_zone %s unchanged, new lease, wait", zname);
6636                 }
6637                 xfr_probe_disown(xfr);
6638                 if(xfr->have_zone)
6639                         xfr->lease_time = *env->now;
6640                 if(xfr->task_nextprobe->worker == NULL)
6641                         xfr_set_timeout(xfr, env, 0, 0);
6642         } else {
6643                 if(verbosity >= VERB_ALGO) {
6644                         char zname[255+1];
6645                         dname_str(xfr->name, zname);
6646                         verbose(VERB_ALGO, "auth zone %s soa probe failed, wait to retry", zname);
6647                 }
6648                 /* we failed to send this as well, move to the wait task,
6649                  * use the shorter retry timeout */
6650                 xfr_probe_disown(xfr);
6651                 /* pick up the nextprobe task and wait */
6652                 if(xfr->task_nextprobe->worker == NULL)
6653                         xfr_set_timeout(xfr, env, 1, 0);
6654         }
6655
6656         lock_basic_unlock(&xfr->lock);
6657 }
6658
6659 /** callback for task_probe lookup of host name, of A or AAAA */
6660 void auth_xfer_probe_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
6661         enum sec_status ATTR_UNUSED(sec), char* ATTR_UNUSED(why_bogus),
6662         int ATTR_UNUSED(was_ratelimited))
6663 {
6664         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6665         struct module_env* env;
6666         log_assert(xfr->task_probe);
6667         lock_basic_lock(&xfr->lock);
6668         env = xfr->task_probe->env;
6669         if(!env || env->outnet->want_to_quit) {
6670                 lock_basic_unlock(&xfr->lock);
6671                 return; /* stop on quit */
6672         }
6673
6674         /* process result */
6675         if(rcode == LDNS_RCODE_NOERROR) {
6676                 uint16_t wanted_qtype = LDNS_RR_TYPE_A;
6677                 struct regional* temp = env->scratch;
6678                 struct query_info rq;
6679                 struct reply_info* rep;
6680                 if(xfr->task_probe->lookup_aaaa)
6681                         wanted_qtype = LDNS_RR_TYPE_AAAA;
6682                 memset(&rq, 0, sizeof(rq));
6683                 rep = parse_reply_in_temp_region(buf, temp, &rq);
6684                 if(rep && rq.qtype == wanted_qtype &&
6685                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
6686                         /* parsed successfully */
6687                         struct ub_packed_rrset_key* answer =
6688                                 reply_find_answer_rrset(&rq, rep);
6689                         if(answer) {
6690                                 xfr_master_add_addrs(xfr->task_probe->
6691                                         lookup_target, answer, wanted_qtype);
6692                         } else {
6693                                 if(verbosity >= VERB_ALGO) {
6694                                         char zname[255+1];
6695                                         dname_str(xfr->name, zname);
6696                                         verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has nodata", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6697                                 }
6698                         }
6699                 } else {
6700                         if(verbosity >= VERB_ALGO) {
6701                                 char zname[255+1];
6702                                 dname_str(xfr->name, zname);
6703                                 verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has no address", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6704                         }
6705                 }
6706                 regional_free_all(temp);
6707         } else {
6708                 if(verbosity >= VERB_ALGO) {
6709                         char zname[255+1];
6710                         dname_str(xfr->name, zname);
6711                         verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup failed", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
6712                 }
6713         }
6714         if(xfr->task_probe->lookup_target->list &&
6715                 xfr->task_probe->lookup_target == xfr_probe_current_master(xfr))
6716                 xfr->task_probe->scan_addr = xfr->task_probe->lookup_target->list;
6717
6718         /* move to lookup AAAA after A lookup, move to next hostname lookup,
6719          * or move to send the probes, or, if nothing to do, end task_probe */
6720         xfr_probe_move_to_next_lookup(xfr, env);
6721         xfr_probe_send_or_end(xfr, env);
6722 }
6723
6724 /** disown task_nextprobe.  caller must hold xfr.lock */
6725 static void
6726 xfr_nextprobe_disown(struct auth_xfer* xfr)
6727 {
6728         /* delete the timer, because the next worker to pick this up may
6729          * not have the same event base */
6730         comm_timer_delete(xfr->task_nextprobe->timer);
6731         xfr->task_nextprobe->timer = NULL;
6732         xfr->task_nextprobe->next_probe = 0;
6733         /* we don't own this item anymore */
6734         xfr->task_nextprobe->worker = NULL;
6735         xfr->task_nextprobe->env = NULL;
6736 }
6737
6738 /** xfer nextprobe timeout callback, this is part of task_nextprobe */
6739 void
6740 auth_xfer_timer(void* arg)
6741 {
6742         struct auth_xfer* xfr = (struct auth_xfer*)arg;
6743         struct module_env* env;
6744         log_assert(xfr->task_nextprobe);
6745         lock_basic_lock(&xfr->lock);
6746         env = xfr->task_nextprobe->env;
6747         if(!env || env->outnet->want_to_quit) {
6748                 lock_basic_unlock(&xfr->lock);
6749                 return; /* stop on quit */
6750         }
6751
6752         /* see if zone has expired, and if so, also set auth_zone expired */
6753         if(xfr->have_zone && !xfr->zone_expired &&
6754            *env->now >= xfr->lease_time + xfr->expiry) {
6755                 lock_basic_unlock(&xfr->lock);
6756                 auth_xfer_set_expired(xfr, env, 1);
6757                 lock_basic_lock(&xfr->lock);
6758         }
6759
6760         xfr_nextprobe_disown(xfr);
6761
6762         if(!xfr_start_probe(xfr, env, NULL)) {
6763                 /* not started because already in progress */
6764                 lock_basic_unlock(&xfr->lock);
6765         }
6766 }
6767
6768 /** return true if there are probe (SOA UDP query) targets in the master list*/
6769 static int
6770 have_probe_targets(struct auth_master* list)
6771 {
6772         struct auth_master* p;
6773         for(p=list; p; p = p->next) {
6774                 if(!p->allow_notify && p->host)
6775                         return 1;
6776         }
6777         return 0;
6778 }
6779
6780 /** start task_probe if possible, if no masters for probe start task_transfer
6781  * returns true if task has been started, and false if the task is already
6782  * in progress. */
6783 static int
6784 xfr_start_probe(struct auth_xfer* xfr, struct module_env* env,
6785         struct auth_master* spec)
6786 {
6787         /* see if we need to start a probe (or maybe it is already in
6788          * progress (due to notify)) */
6789         if(xfr->task_probe->worker == NULL) {
6790                 if(!have_probe_targets(xfr->task_probe->masters) &&
6791                         !(xfr->task_probe->only_lookup &&
6792                         xfr->task_probe->masters != NULL)) {
6793                         /* useless to pick up task_probe, no masters to
6794                          * probe. Instead attempt to pick up task transfer */
6795                         if(xfr->task_transfer->worker == NULL) {
6796                                 xfr_start_transfer(xfr, env, spec);
6797                                 return 1;
6798                         }
6799                         /* task transfer already in progress */
6800                         return 0;
6801                 }
6802
6803                 /* pick up the probe task ourselves */
6804                 xfr->task_probe->worker = env->worker;
6805                 xfr->task_probe->env = env;
6806                 xfr->task_probe->cp = NULL;
6807
6808                 /* start the task */
6809                 /* have not seen a new lease yet, this scan */
6810                 xfr->task_probe->have_new_lease = 0;
6811                 /* if this was a timeout, no specific first master to scan */
6812                 /* otherwise, spec is nonNULL the notified master, scan
6813                  * first and also transfer first from it */
6814                 xfr_probe_start_list(xfr, spec);
6815                 /* setup to start the lookup of hostnames of masters afresh */
6816                 xfr_probe_start_lookups(xfr);
6817                 /* send the probe packet or next send, or end task */
6818                 xfr_probe_send_or_end(xfr, env);
6819                 return 1;
6820         }
6821         return 0;
6822 }
6823
6824 /** for task_nextprobe.
6825  * determine next timeout for auth_xfer. Also (re)sets timer.
6826  * @param xfr: task structure
6827  * @param env: module environment, with worker and time.
6828  * @param failure: set true if timer should be set for failure retry.
6829  * @param lookup_only: only perform lookups when timer done, 0 sec timeout
6830  */
6831 static void
6832 xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
6833         int failure, int lookup_only)
6834 {
6835         struct timeval tv;
6836         log_assert(xfr->task_nextprobe != NULL);
6837         log_assert(xfr->task_nextprobe->worker == NULL ||
6838                 xfr->task_nextprobe->worker == env->worker);
6839         /* normally, nextprobe = startoflease + refresh,
6840          * but if expiry is sooner, use that one.
6841          * after a failure, use the retry timer instead. */
6842         xfr->task_nextprobe->next_probe = *env->now;
6843         if(xfr->lease_time && !failure)
6844                 xfr->task_nextprobe->next_probe = xfr->lease_time;
6845
6846         if(!failure) {
6847                 xfr->task_nextprobe->backoff = 0;
6848         } else {
6849                 if(xfr->task_nextprobe->backoff == 0)
6850                                 xfr->task_nextprobe->backoff = 3;
6851                 else    xfr->task_nextprobe->backoff *= 2;
6852                 if(xfr->task_nextprobe->backoff > AUTH_TRANSFER_MAX_BACKOFF)
6853                         xfr->task_nextprobe->backoff =
6854                                 AUTH_TRANSFER_MAX_BACKOFF;
6855         }
6856
6857         if(xfr->have_zone) {
6858                 time_t wait = xfr->refresh;
6859                 if(failure) wait = xfr->retry;
6860                 if(xfr->expiry < wait)
6861                         xfr->task_nextprobe->next_probe += xfr->expiry;
6862                 else    xfr->task_nextprobe->next_probe += wait;
6863                 if(failure)
6864                         xfr->task_nextprobe->next_probe +=
6865                                 xfr->task_nextprobe->backoff;
6866                 /* put the timer exactly on expiry, if possible */
6867                 if(xfr->lease_time && xfr->lease_time+xfr->expiry <
6868                         xfr->task_nextprobe->next_probe &&
6869                         xfr->lease_time+xfr->expiry > *env->now)
6870                         xfr->task_nextprobe->next_probe =
6871                                 xfr->lease_time+xfr->expiry;
6872         } else {
6873                 xfr->task_nextprobe->next_probe +=
6874                         xfr->task_nextprobe->backoff;
6875         }
6876
6877         if(!xfr->task_nextprobe->timer) {
6878                 xfr->task_nextprobe->timer = comm_timer_create(
6879                         env->worker_base, auth_xfer_timer, xfr);
6880                 if(!xfr->task_nextprobe->timer) {
6881                         /* failed to malloc memory. likely zone transfer
6882                          * also fails for that. skip the timeout */
6883                         char zname[255+1];
6884                         dname_str(xfr->name, zname);
6885                         log_err("cannot allocate timer, no refresh for %s",
6886                                 zname);
6887                         return;
6888                 }
6889         }
6890         xfr->task_nextprobe->worker = env->worker;
6891         xfr->task_nextprobe->env = env;
6892         if(*(xfr->task_nextprobe->env->now) <= xfr->task_nextprobe->next_probe)
6893                 tv.tv_sec = xfr->task_nextprobe->next_probe -
6894                         *(xfr->task_nextprobe->env->now);
6895         else    tv.tv_sec = 0;
6896         if(tv.tv_sec != 0 && lookup_only && xfr->task_probe->masters) {
6897                 /* don't lookup_only, if lookup timeout is 0 anyway,
6898                  * or if we don't have masters to lookup */
6899                 tv.tv_sec = 0;
6900                 if(xfr->task_probe->worker == NULL)
6901                         xfr->task_probe->only_lookup = 1;
6902         }
6903         if(verbosity >= VERB_ALGO) {
6904                 char zname[255+1];
6905                 dname_str(xfr->name, zname);
6906                 verbose(VERB_ALGO, "auth zone %s timeout in %d seconds",
6907                         zname, (int)tv.tv_sec);
6908         }
6909         tv.tv_usec = 0;
6910         comm_timer_set(xfr->task_nextprobe->timer, &tv);
6911 }
6912
6913 /** initial pick up of worker timeouts, ties events to worker event loop */
6914 void
6915 auth_xfer_pickup_initial(struct auth_zones* az, struct module_env* env)
6916 {
6917         struct auth_xfer* x;
6918         lock_rw_wrlock(&az->lock);
6919         RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6920                 lock_basic_lock(&x->lock);
6921                 /* set lease_time, because we now have timestamp in env,
6922                  * (not earlier during startup and apply_cfg), and this
6923                  * notes the start time when the data was acquired */
6924                 if(x->have_zone)
6925                         x->lease_time = *env->now;
6926                 if(x->task_nextprobe && x->task_nextprobe->worker == NULL) {
6927                         xfr_set_timeout(x, env, 0, 1);
6928                 }
6929                 lock_basic_unlock(&x->lock);
6930         }
6931         lock_rw_unlock(&az->lock);
6932 }
6933
6934 void auth_zones_cleanup(struct auth_zones* az)
6935 {
6936         struct auth_xfer* x;
6937         lock_rw_wrlock(&az->lock);
6938         RBTREE_FOR(x, struct auth_xfer*, &az->xtree) {
6939                 lock_basic_lock(&x->lock);
6940                 if(x->task_nextprobe && x->task_nextprobe->worker != NULL) {
6941                         xfr_nextprobe_disown(x);
6942                 }
6943                 if(x->task_probe && x->task_probe->worker != NULL) {
6944                         xfr_probe_disown(x);
6945                 }
6946                 if(x->task_transfer && x->task_transfer->worker != NULL) {
6947                         auth_chunks_delete(x->task_transfer);
6948                         xfr_transfer_disown(x);
6949                 }
6950                 lock_basic_unlock(&x->lock);
6951         }
6952         lock_rw_unlock(&az->lock);
6953 }
6954
6955 /**
6956  * malloc the xfer and tasks
6957  * @param z: auth_zone with name of zone.
6958  */
6959 static struct auth_xfer*
6960 auth_xfer_new(struct auth_zone* z)
6961 {
6962         struct auth_xfer* xfr;
6963         xfr = (struct auth_xfer*)calloc(1, sizeof(*xfr));
6964         if(!xfr) return NULL;
6965         xfr->name = memdup(z->name, z->namelen);
6966         if(!xfr->name) {
6967                 free(xfr);
6968                 return NULL;
6969         }
6970         xfr->node.key = xfr;
6971         xfr->namelen = z->namelen;
6972         xfr->namelabs = z->namelabs;
6973         xfr->dclass = z->dclass;
6974
6975         xfr->task_nextprobe = (struct auth_nextprobe*)calloc(1,
6976                 sizeof(struct auth_nextprobe));
6977         if(!xfr->task_nextprobe) {
6978                 free(xfr->name);
6979                 free(xfr);
6980                 return NULL;
6981         }
6982         xfr->task_probe = (struct auth_probe*)calloc(1,
6983                 sizeof(struct auth_probe));
6984         if(!xfr->task_probe) {
6985                 free(xfr->task_nextprobe);
6986                 free(xfr->name);
6987                 free(xfr);
6988                 return NULL;
6989         }
6990         xfr->task_transfer = (struct auth_transfer*)calloc(1,
6991                 sizeof(struct auth_transfer));
6992         if(!xfr->task_transfer) {
6993                 free(xfr->task_probe);
6994                 free(xfr->task_nextprobe);
6995                 free(xfr->name);
6996                 free(xfr);
6997                 return NULL;
6998         }
6999
7000         lock_basic_init(&xfr->lock);
7001         lock_protect(&xfr->lock, &xfr->name, sizeof(xfr->name));
7002         lock_protect(&xfr->lock, &xfr->namelen, sizeof(xfr->namelen));
7003         lock_protect(&xfr->lock, xfr->name, xfr->namelen);
7004         lock_protect(&xfr->lock, &xfr->namelabs, sizeof(xfr->namelabs));
7005         lock_protect(&xfr->lock, &xfr->dclass, sizeof(xfr->dclass));
7006         lock_protect(&xfr->lock, &xfr->notify_received, sizeof(xfr->notify_received));
7007         lock_protect(&xfr->lock, &xfr->notify_serial, sizeof(xfr->notify_serial));
7008         lock_protect(&xfr->lock, &xfr->zone_expired, sizeof(xfr->zone_expired));
7009         lock_protect(&xfr->lock, &xfr->have_zone, sizeof(xfr->have_zone));
7010         lock_protect(&xfr->lock, &xfr->serial, sizeof(xfr->serial));
7011         lock_protect(&xfr->lock, &xfr->retry, sizeof(xfr->retry));
7012         lock_protect(&xfr->lock, &xfr->refresh, sizeof(xfr->refresh));
7013         lock_protect(&xfr->lock, &xfr->expiry, sizeof(xfr->expiry));
7014         lock_protect(&xfr->lock, &xfr->lease_time, sizeof(xfr->lease_time));
7015         lock_protect(&xfr->lock, &xfr->task_nextprobe->worker,
7016                 sizeof(xfr->task_nextprobe->worker));
7017         lock_protect(&xfr->lock, &xfr->task_probe->worker,
7018                 sizeof(xfr->task_probe->worker));
7019         lock_protect(&xfr->lock, &xfr->task_transfer->worker,
7020                 sizeof(xfr->task_transfer->worker));
7021         lock_basic_lock(&xfr->lock);
7022         return xfr;
7023 }
7024
7025 /** Create auth_xfer structure.
7026  * This populates the have_zone, soa values, and so on times.
7027  * and sets the timeout, if a zone transfer is needed a short timeout is set.
7028  * For that the auth_zone itself must exist (and read in zonefile)
7029  * returns false on alloc failure. */
7030 struct auth_xfer*
7031 auth_xfer_create(struct auth_zones* az, struct auth_zone* z)
7032 {
7033         struct auth_xfer* xfr;
7034
7035         /* malloc it */
7036         xfr = auth_xfer_new(z);
7037         if(!xfr) {
7038                 log_err("malloc failure");
7039                 return NULL;
7040         }
7041         /* insert in tree */
7042         (void)rbtree_insert(&az->xtree, &xfr->node);
7043         return xfr;
7044 }
7045
7046 /** create new auth_master structure */
7047 static struct auth_master*
7048 auth_master_new(struct auth_master*** list)
7049 {
7050         struct auth_master *m;
7051         m = (struct auth_master*)calloc(1, sizeof(*m));
7052         if(!m) {
7053                 log_err("malloc failure");
7054                 return NULL;
7055         }
7056         /* set first pointer to m, or next pointer of previous element to m */
7057         (**list) = m;
7058         /* store m's next pointer as future point to store at */
7059         (*list) = &(m->next);
7060         return m;
7061 }
7062
7063 /** dup_prefix : create string from initial part of other string, malloced */
7064 static char*
7065 dup_prefix(char* str, size_t num)
7066 {
7067         char* result;
7068         size_t len = strlen(str);
7069         if(len < num) num = len; /* not more than strlen */
7070         result = (char*)malloc(num+1);
7071         if(!result) {
7072                 log_err("malloc failure");
7073                 return result;
7074         }
7075         memmove(result, str, num);
7076         result[num] = 0;
7077         return result;
7078 }
7079
7080 /** dup string and print error on error */
7081 static char*
7082 dup_all(char* str)
7083 {
7084         char* result = strdup(str);
7085         if(!result) {
7086                 log_err("malloc failure");
7087                 return NULL;
7088         }
7089         return result;
7090 }
7091
7092 /** find first of two characters */
7093 static char*
7094 str_find_first_of_chars(char* s, char a, char b)
7095 {
7096         char* ra = strchr(s, a);
7097         char* rb = strchr(s, b);
7098         if(!ra) return rb;
7099         if(!rb) return ra;
7100         if(ra < rb) return ra;
7101         return rb;
7102 }
7103
7104 /** parse URL into host and file parts, false on malloc or parse error */
7105 static int
7106 parse_url(char* url, char** host, char** file, int* port, int* ssl)
7107 {
7108         char* p = url;
7109         /* parse http://www.example.com/file.htm
7110          * or http://127.0.0.1   (index.html)
7111          * or https://[::1@1234]/a/b/c/d */
7112         *ssl = 1;
7113         *port = AUTH_HTTPS_PORT;
7114
7115         /* parse http:// or https:// */
7116         if(strncmp(p, "http://", 7) == 0) {
7117                 p += 7;
7118                 *ssl = 0;
7119                 *port = AUTH_HTTP_PORT;
7120         } else if(strncmp(p, "https://", 8) == 0) {
7121                 p += 8;
7122         } else if(strstr(p, "://") && strchr(p, '/') > strstr(p, "://") &&
7123                 strchr(p, ':') >= strstr(p, "://")) {
7124                 char* uri = dup_prefix(p, (size_t)(strstr(p, "://")-p));
7125                 log_err("protocol %s:// not supported (for url %s)",
7126                         uri?uri:"", p);
7127                 free(uri);
7128                 return 0;
7129         }
7130
7131         /* parse hostname part */
7132         if(p[0] == '[') {
7133                 char* end = strchr(p, ']');
7134                 p++; /* skip over [ */
7135                 if(end) {
7136                         *host = dup_prefix(p, (size_t)(end-p));
7137                         if(!*host) return 0;
7138                         p = end+1; /* skip over ] */
7139                 } else {
7140                         *host = dup_all(p);
7141                         if(!*host) return 0;
7142                         p = end;
7143                 }
7144         } else {
7145                 char* end = str_find_first_of_chars(p, ':', '/');
7146                 if(end) {
7147                         *host = dup_prefix(p, (size_t)(end-p));
7148                         if(!*host) return 0;
7149                 } else {
7150                         *host = dup_all(p);
7151                         if(!*host) return 0;
7152                 }
7153                 p = end; /* at next : or / or NULL */
7154         }
7155
7156         /* parse port number */
7157         if(p && p[0] == ':') {
7158                 char* end = NULL;
7159                 *port = strtol(p+1, &end, 10);
7160                 p = end;
7161         }
7162
7163         /* parse filename part */
7164         while(p && *p == '/')
7165                 p++;
7166         if(!p || p[0] == 0)
7167                 *file = strdup("/");
7168         else    *file = strdup(p);
7169         if(!*file) {
7170                 log_err("malloc failure");
7171                 return 0;
7172         }
7173         return 1;
7174 }
7175
7176 int
7177 xfer_set_masters(struct auth_master** list, struct config_auth* c,
7178         int with_http)
7179 {
7180         struct auth_master* m;
7181         struct config_strlist* p;
7182         /* list points to the first, or next pointer for the new element */
7183         while(*list) {
7184                 list = &( (*list)->next );
7185         }
7186         if(with_http)
7187           for(p = c->urls; p; p = p->next) {
7188                 m = auth_master_new(&list);
7189                 if(!m) return 0;
7190                 m->http = 1;
7191                 if(!parse_url(p->str, &m->host, &m->file, &m->port, &m->ssl))
7192                         return 0;
7193         }
7194         for(p = c->masters; p; p = p->next) {
7195                 m = auth_master_new(&list);
7196                 if(!m) return 0;
7197                 m->ixfr = 1; /* this flag is not configurable */
7198                 m->host = strdup(p->str);
7199                 if(!m->host) {
7200                         log_err("malloc failure");
7201                         return 0;
7202                 }
7203         }
7204         for(p = c->allow_notify; p; p = p->next) {
7205                 m = auth_master_new(&list);
7206                 if(!m) return 0;
7207                 m->allow_notify = 1;
7208                 m->host = strdup(p->str);
7209                 if(!m->host) {
7210                         log_err("malloc failure");
7211                         return 0;
7212                 }
7213         }
7214         return 1;
7215 }
7216
7217 #define SERIAL_BITS     32
7218 int
7219 compare_serial(uint32_t a, uint32_t b)
7220 {
7221         const uint32_t cutoff = ((uint32_t) 1 << (SERIAL_BITS - 1));
7222
7223         if (a == b) {
7224                 return 0;
7225         } else if ((a < b && b - a < cutoff) || (a > b && a - b > cutoff)) {
7226                 return -1;
7227         } else {
7228                 return 1;
7229         }
7230 }
7231
7232 int zonemd_hashalgo_supported(int hashalgo)
7233 {
7234         if(hashalgo == ZONEMD_ALGO_SHA384) return 1;
7235         if(hashalgo == ZONEMD_ALGO_SHA512) return 1;
7236         return 0;
7237 }
7238
7239 int zonemd_scheme_supported(int scheme)
7240 {
7241         if(scheme == ZONEMD_SCHEME_SIMPLE) return 1;
7242         return 0;
7243 }
7244
7245 /** initialize hash for hashing with zonemd hash algo */
7246 static struct secalgo_hash* zonemd_digest_init(int hashalgo, char** reason)
7247 {
7248         struct secalgo_hash *h;
7249         if(hashalgo == ZONEMD_ALGO_SHA384) {
7250                 /* sha384 */
7251                 h = secalgo_hash_create_sha384();
7252                 if(!h)
7253                         *reason = "digest sha384 could not be created";
7254                 return h;
7255         } else if(hashalgo == ZONEMD_ALGO_SHA512) {
7256                 /* sha512 */
7257                 h = secalgo_hash_create_sha512();
7258                 if(!h)
7259                         *reason = "digest sha512 could not be created";
7260                 return h;
7261         }
7262         /* unknown hash algo */
7263         *reason = "unsupported algorithm";
7264         return NULL;
7265 }
7266
7267 /** update the hash for zonemd */
7268 static int zonemd_digest_update(int hashalgo, struct secalgo_hash* h,
7269         uint8_t* data, size_t len, char** reason)
7270 {
7271         if(hashalgo == ZONEMD_ALGO_SHA384) {
7272                 if(!secalgo_hash_update(h, data, len)) {
7273                         *reason = "digest sha384 failed";
7274                         return 0;
7275                 }
7276                 return 1;
7277         } else if(hashalgo == ZONEMD_ALGO_SHA512) {
7278                 if(!secalgo_hash_update(h, data, len)) {
7279                         *reason = "digest sha512 failed";
7280                         return 0;
7281                 }
7282                 return 1;
7283         }
7284         /* unknown hash algo */
7285         *reason = "unsupported algorithm";
7286         return 0;
7287 }
7288
7289 /** finish the hash for zonemd */
7290 static int zonemd_digest_finish(int hashalgo, struct secalgo_hash* h,
7291         uint8_t* result, size_t hashlen, size_t* resultlen, char** reason)
7292 {
7293         if(hashalgo == ZONEMD_ALGO_SHA384) {
7294                 if(hashlen < 384/8) {
7295                         *reason = "digest buffer too small for sha384";
7296                         return 0;
7297                 }
7298                 if(!secalgo_hash_final(h, result, hashlen, resultlen)) {
7299                         *reason = "digest sha384 finish failed";
7300                         return 0;
7301                 }
7302                 return 1;
7303         } else if(hashalgo == ZONEMD_ALGO_SHA512) {
7304                 if(hashlen < 512/8) {
7305                         *reason = "digest buffer too small for sha512";
7306                         return 0;
7307                 }
7308                 if(!secalgo_hash_final(h, result, hashlen, resultlen)) {
7309                         *reason = "digest sha512 finish failed";
7310                         return 0;
7311                 }
7312                 return 1;
7313         }
7314         /* unknown algo */
7315         *reason = "unsupported algorithm";
7316         return 0;
7317 }
7318
7319 /** add rrsets from node to the list */
7320 static size_t authdata_rrsets_to_list(struct auth_rrset** array,
7321         size_t arraysize, struct auth_rrset* first)
7322 {
7323         struct auth_rrset* rrset = first;
7324         size_t num = 0;
7325         while(rrset) {
7326                 if(num >= arraysize)
7327                         return num;
7328                 array[num] = rrset;
7329                 num++;
7330                 rrset = rrset->next;
7331         }
7332         return num;
7333 }
7334
7335 /** compare rr list entries */
7336 static int rrlist_compare(const void* arg1, const void* arg2)
7337 {
7338         struct auth_rrset* r1 = *(struct auth_rrset**)arg1;
7339         struct auth_rrset* r2 = *(struct auth_rrset**)arg2;
7340         uint16_t t1, t2;
7341         if(r1 == NULL) t1 = LDNS_RR_TYPE_RRSIG;
7342         else t1 = r1->type;
7343         if(r2 == NULL) t2 = LDNS_RR_TYPE_RRSIG;
7344         else t2 = r2->type;
7345         if(t1 < t2)
7346                 return -1;
7347         if(t1 > t2)
7348                 return 1;
7349         return 0;
7350 }
7351
7352 /** add type RRSIG to rr list if not one there already,
7353  * this is to perform RRSIG collate processing at that point. */
7354 static void addrrsigtype_if_needed(struct auth_rrset** array,
7355         size_t arraysize, size_t* rrnum, struct auth_data* node)
7356 {
7357         if(az_domain_rrset(node, LDNS_RR_TYPE_RRSIG))
7358                 return; /* already one there */
7359         if((*rrnum) >= arraysize)
7360                 return; /* array too small? */
7361         array[*rrnum] = NULL; /* nothing there, but need entry in list */
7362         (*rrnum)++;
7363 }
7364
7365 /** collate the RRs in an RRset using the simple scheme */
7366 static int zonemd_simple_rrset(struct auth_zone* z, int hashalgo,
7367         struct secalgo_hash* h, struct auth_data* node,
7368         struct auth_rrset* rrset, struct regional* region,
7369         struct sldns_buffer* buf, char** reason)
7370 {
7371         /* canonicalize */
7372         struct ub_packed_rrset_key key;
7373         memset(&key, 0, sizeof(key));
7374         key.entry.key = &key;
7375         key.entry.data = rrset->data;
7376         key.rk.dname = node->name;
7377         key.rk.dname_len = node->namelen;
7378         key.rk.type = htons(rrset->type);
7379         key.rk.rrset_class = htons(z->dclass);
7380         if(!rrset_canonicalize_to_buffer(region, buf, &key)) {
7381                 *reason = "out of memory";
7382                 return 0;
7383         }
7384         regional_free_all(region);
7385
7386         /* hash */
7387         if(!zonemd_digest_update(hashalgo, h, sldns_buffer_begin(buf),
7388                 sldns_buffer_limit(buf), reason)) {
7389                 return 0;
7390         }
7391         return 1;
7392 }
7393
7394 /** count number of RRSIGs in a domain name rrset list */
7395 static size_t zonemd_simple_count_rrsig(struct auth_rrset* rrset,
7396         struct auth_rrset** rrlist, size_t rrnum,
7397         struct auth_zone* z, struct auth_data* node)
7398 {
7399         size_t i, count = 0;
7400         if(rrset) {
7401                 size_t j;
7402                 for(j = 0; j<rrset->data->count; j++) {
7403                         if(rrsig_rdata_get_type_covered(rrset->data->
7404                                 rr_data[j], rrset->data->rr_len[j]) ==
7405                                 LDNS_RR_TYPE_ZONEMD &&
7406                                 query_dname_compare(z->name, node->name)==0) {
7407                                 /* omit RRSIGs over type ZONEMD at apex */
7408                                 continue;
7409                         }
7410                         count++;
7411                 }
7412         }
7413         for(i=0; i<rrnum; i++) {
7414                 if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7415                         query_dname_compare(z->name, node->name)==0) {
7416                         /* omit RRSIGs over type ZONEMD at apex */
7417                         continue;
7418                 }
7419                 count += (rrlist[i]?rrlist[i]->data->rrsig_count:0);
7420         }
7421         return count;
7422 }
7423
7424 /** allocate sparse rrset data for the number of entries in tepm region */
7425 static int zonemd_simple_rrsig_allocs(struct regional* region,
7426         struct packed_rrset_data* data, size_t count)
7427 {
7428         data->rr_len = regional_alloc(region, sizeof(*data->rr_len) * count);
7429         if(!data->rr_len) {
7430                 return 0;
7431         }
7432         data->rr_ttl = regional_alloc(region, sizeof(*data->rr_ttl) * count);
7433         if(!data->rr_ttl) {
7434                 return 0;
7435         }
7436         data->rr_data = regional_alloc(region, sizeof(*data->rr_data) * count);
7437         if(!data->rr_data) {
7438                 return 0;
7439         }
7440         return 1;
7441 }
7442
7443 /** add the RRSIGs from the rrs in the domain into the data */
7444 static void add_rrlist_rrsigs_into_data(struct packed_rrset_data* data,
7445         size_t* done, struct auth_rrset** rrlist, size_t rrnum,
7446         struct auth_zone* z, struct auth_data* node)
7447 {
7448         size_t i;
7449         for(i=0; i<rrnum; i++) {
7450                 size_t j;
7451                 if(!rrlist[i])
7452                         continue;
7453                 if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7454                         query_dname_compare(z->name, node->name)==0) {
7455                         /* omit RRSIGs over type ZONEMD at apex */
7456                         continue;
7457                 }
7458                 for(j = 0; j<rrlist[i]->data->rrsig_count; j++) {
7459                         data->rr_len[*done] = rrlist[i]->data->rr_len[rrlist[i]->data->count + j];
7460                         data->rr_ttl[*done] = rrlist[i]->data->rr_ttl[rrlist[i]->data->count + j];
7461                         /* reference the rdata in the rrset, no need to
7462                          * copy it, it is no longer needed at the end of
7463                          * the routine */
7464                         data->rr_data[*done] = rrlist[i]->data->rr_data[rrlist[i]->data->count + j];
7465                         (*done)++;
7466                 }
7467         }
7468 }
7469
7470 static void add_rrset_into_data(struct packed_rrset_data* data,
7471         size_t* done, struct auth_rrset* rrset,
7472         struct auth_zone* z, struct auth_data* node)
7473 {
7474         if(rrset) {
7475                 size_t j;
7476                 for(j = 0; j<rrset->data->count; j++) {
7477                         if(rrsig_rdata_get_type_covered(rrset->data->
7478                                 rr_data[j], rrset->data->rr_len[j]) ==
7479                                 LDNS_RR_TYPE_ZONEMD &&
7480                                 query_dname_compare(z->name, node->name)==0) {
7481                                 /* omit RRSIGs over type ZONEMD at apex */
7482                                 continue;
7483                         }
7484                         data->rr_len[*done] = rrset->data->rr_len[j];
7485                         data->rr_ttl[*done] = rrset->data->rr_ttl[j];
7486                         /* reference the rdata in the rrset, no need to
7487                          * copy it, it is no longer need at the end of
7488                          * the routine */
7489                         data->rr_data[*done] = rrset->data->rr_data[j];
7490                         (*done)++;
7491                 }
7492         }
7493 }
7494
7495 /** collate the RRSIGs using the simple scheme */
7496 static int zonemd_simple_rrsig(struct auth_zone* z, int hashalgo,
7497         struct secalgo_hash* h, struct auth_data* node,
7498         struct auth_rrset* rrset, struct auth_rrset** rrlist, size_t rrnum,
7499         struct regional* region, struct sldns_buffer* buf, char** reason)
7500 {
7501         /* the rrset pointer can be NULL, this means it is type RRSIG and
7502          * there is no ordinary type RRSIG there.  The RRSIGs are stored
7503          * with the RRsets in their data.
7504          *
7505          * The RRset pointer can be nonNULL. This happens if there is
7506          * no RR that is covered by the RRSIG for the domain.  Then this
7507          * RRSIG RR is stored in an rrset of type RRSIG. The other RRSIGs
7508          * are stored in the rrset entries for the RRs in the rr list for
7509          * the domain node.  We need to collate the rrset's data, if any, and
7510          * the rrlist's rrsigs */
7511         /* if this is the apex, omit RRSIGs that cover type ZONEMD */
7512         /* build rrsig rrset */
7513         size_t done = 0;
7514         struct ub_packed_rrset_key key;
7515         struct packed_rrset_data data;
7516         memset(&key, 0, sizeof(key));
7517         memset(&data, 0, sizeof(data));
7518         key.entry.key = &key;
7519         key.entry.data = &data;
7520         key.rk.dname = node->name;
7521         key.rk.dname_len = node->namelen;
7522         key.rk.type = htons(LDNS_RR_TYPE_RRSIG);
7523         key.rk.rrset_class = htons(z->dclass);
7524         data.count = zonemd_simple_count_rrsig(rrset, rrlist, rrnum, z, node);
7525         if(!zonemd_simple_rrsig_allocs(region, &data, data.count)) {
7526                 *reason = "out of memory";
7527                 regional_free_all(region);
7528                 return 0;
7529         }
7530         /* all the RRSIGs stored in the other rrsets for this domain node */
7531         add_rrlist_rrsigs_into_data(&data, &done, rrlist, rrnum, z, node);
7532         /* plus the RRSIGs stored in an rrset of type RRSIG for this node */
7533         add_rrset_into_data(&data, &done, rrset, z, node);
7534
7535         /* canonicalize */
7536         if(!rrset_canonicalize_to_buffer(region, buf, &key)) {
7537                 *reason = "out of memory";
7538                 regional_free_all(region);
7539                 return 0;
7540         }
7541         regional_free_all(region);
7542
7543         /* hash */
7544         if(!zonemd_digest_update(hashalgo, h, sldns_buffer_begin(buf),
7545                 sldns_buffer_limit(buf), reason)) {
7546                 return 0;
7547         }
7548         return 1;
7549 }
7550
7551 /** collate a domain's rrsets using the simple scheme */
7552 static int zonemd_simple_domain(struct auth_zone* z, int hashalgo,
7553         struct secalgo_hash* h, struct auth_data* node,
7554         struct regional* region, struct sldns_buffer* buf, char** reason)
7555 {
7556         const size_t rrlistsize = 65536;
7557         struct auth_rrset* rrlist[rrlistsize];
7558         size_t i, rrnum = 0;
7559         /* see if the domain is out of scope, the zone origin,
7560          * that would be omitted */
7561         if(!dname_subdomain_c(node->name, z->name))
7562                 return 1; /* continue */
7563         /* loop over the rrsets in ascending order. */
7564         rrnum = authdata_rrsets_to_list(rrlist, rrlistsize, node->rrsets);
7565         addrrsigtype_if_needed(rrlist, rrlistsize, &rrnum, node);
7566         qsort(rrlist, rrnum, sizeof(*rrlist), rrlist_compare);
7567         for(i=0; i<rrnum; i++) {
7568                 if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
7569                         query_dname_compare(z->name, node->name) == 0) {
7570                         /* omit type ZONEMD at apex */
7571                         continue;
7572                 }
7573                 if(rrlist[i] == NULL || rrlist[i]->type ==
7574                         LDNS_RR_TYPE_RRSIG) {
7575                         if(!zonemd_simple_rrsig(z, hashalgo, h, node,
7576                                 rrlist[i], rrlist, rrnum, region, buf, reason))
7577                                 return 0;
7578                 } else if(!zonemd_simple_rrset(z, hashalgo, h, node,
7579                         rrlist[i], region, buf, reason)) {
7580                         return 0;
7581                 }
7582         }
7583         return 1;
7584 }
7585
7586 /** collate the zone using the simple scheme */
7587 static int zonemd_simple_collate(struct auth_zone* z, int hashalgo,
7588         struct secalgo_hash* h, struct regional* region,
7589         struct sldns_buffer* buf, char** reason)
7590 {
7591         /* our tree is sorted in canonical order, so we can just loop over
7592          * the tree */
7593         struct auth_data* n;
7594         RBTREE_FOR(n, struct auth_data*, &z->data) {
7595                 if(!zonemd_simple_domain(z, hashalgo, h, n, region, buf,
7596                         reason))
7597                         return 0;
7598         }
7599         return 1;
7600 }
7601
7602 int auth_zone_generate_zonemd_hash(struct auth_zone* z, int scheme,
7603         int hashalgo, uint8_t* hash, size_t hashlen, size_t* resultlen,
7604         struct regional* region, struct sldns_buffer* buf, char** reason)
7605 {
7606         struct secalgo_hash* h = zonemd_digest_init(hashalgo, reason);
7607         if(!h) {
7608                 if(!*reason)
7609                         *reason = "digest init fail";
7610                 return 0;
7611         }
7612         if(scheme == ZONEMD_SCHEME_SIMPLE) {
7613                 if(!zonemd_simple_collate(z, hashalgo, h, region, buf, reason)) {
7614                         if(!*reason) *reason = "scheme simple collate fail";
7615                         secalgo_hash_delete(h);
7616                         return 0;
7617                 }
7618         }
7619         if(!zonemd_digest_finish(hashalgo, h, hash, hashlen, resultlen,
7620                 reason)) {
7621                 secalgo_hash_delete(h);
7622                 *reason = "digest finish fail";
7623                 return 0;
7624         }
7625         secalgo_hash_delete(h);
7626         return 1;
7627 }
7628
7629 int auth_zone_generate_zonemd_check(struct auth_zone* z, int scheme,
7630         int hashalgo, uint8_t* hash, size_t hashlen, struct regional* region,
7631         struct sldns_buffer* buf, char** reason)
7632 {
7633         uint8_t gen[512];
7634         size_t genlen = 0;
7635         if(!zonemd_hashalgo_supported(hashalgo)) {
7636                 *reason = "unsupported algorithm";
7637                 return 0;
7638         }
7639         if(!zonemd_scheme_supported(scheme)) {
7640                 *reason = "unsupported scheme";
7641                 return 0;
7642         }
7643         if(hashlen < 12) {
7644                 /* the ZONEMD draft requires digests to fail if too small */
7645                 *reason = "digest length too small, less than 12";
7646                 return 0;
7647         }
7648         /* generate digest */
7649         if(!auth_zone_generate_zonemd_hash(z, scheme, hashalgo, gen,
7650                 sizeof(gen), &genlen, region, buf, reason)) {
7651                 /* reason filled in by zonemd hash routine */
7652                 return 0;
7653         }
7654         /* check digest length */
7655         if(hashlen != genlen) {
7656                 *reason = "incorrect digest length";
7657                 if(verbosity >= VERB_ALGO) {
7658                         verbose(VERB_ALGO, "zonemd scheme=%d hashalgo=%d",
7659                                 scheme, hashalgo);
7660                         log_hex("ZONEMD should be  ", gen, genlen);
7661                         log_hex("ZONEMD to check is", hash, hashlen);
7662                 }
7663                 return 0;
7664         }
7665         /* check digest */
7666         if(memcmp(hash, gen, genlen) != 0) {
7667                 *reason = "incorrect digest";
7668                 if(verbosity >= VERB_ALGO) {
7669                         verbose(VERB_ALGO, "zonemd scheme=%d hashalgo=%d",
7670                                 scheme, hashalgo);
7671                         log_hex("ZONEMD should be  ", gen, genlen);
7672                         log_hex("ZONEMD to check is", hash, hashlen);
7673                 }
7674                 return 0;
7675         }
7676         return 1;
7677 }
7678
7679 /** log auth zone message with zone name in front. */
7680 static void auth_zone_log(uint8_t* name, enum verbosity_value level,
7681         const char* format, ...) ATTR_FORMAT(printf, 3, 4);
7682 static void auth_zone_log(uint8_t* name, enum verbosity_value level,
7683         const char* format, ...)
7684 {
7685         va_list args;
7686         va_start(args, format);
7687         if(verbosity >= level) {
7688                 char str[255+1];
7689                 char msg[MAXSYSLOGMSGLEN];
7690                 dname_str(name, str);
7691                 vsnprintf(msg, sizeof(msg), format, args);
7692                 verbose(level, "auth zone %s %s", str, msg);
7693         }
7694         va_end(args);
7695 }
7696
7697 /** ZONEMD, dnssec verify the rrset with the dnskey */
7698 static int zonemd_dnssec_verify_rrset(struct auth_zone* z,
7699         struct module_env* env, struct module_stack* mods,
7700         struct ub_packed_rrset_key* dnskey, struct auth_data* node,
7701         struct auth_rrset* rrset, char** why_bogus, uint8_t* sigalg)
7702 {
7703         struct ub_packed_rrset_key pk;
7704         enum sec_status sec;
7705         struct val_env* ve;
7706         int m;
7707         m = modstack_find(mods, "validator");
7708         if(m == -1) {
7709                 auth_zone_log(z->name, VERB_ALGO, "zonemd dnssec verify: have "
7710                         "DNSKEY chain of trust, but no validator module");
7711                 return 0;
7712         }
7713         ve = (struct val_env*)env->modinfo[m];
7714
7715         memset(&pk, 0, sizeof(pk));
7716         pk.entry.key = &pk;
7717         pk.entry.data = rrset->data;
7718         pk.rk.dname = node->name;
7719         pk.rk.dname_len = node->namelen;
7720         pk.rk.type = htons(rrset->type);
7721         pk.rk.rrset_class = htons(z->dclass);
7722         if(verbosity >= VERB_ALGO) {
7723                 char typestr[32];
7724                 typestr[0]=0;
7725                 sldns_wire2str_type_buf(rrset->type, typestr, sizeof(typestr));
7726                 auth_zone_log(z->name, VERB_ALGO,
7727                         "zonemd: verify %s RRset with DNSKEY", typestr);
7728         }
7729         sec = dnskeyset_verify_rrset(env, ve, &pk, dnskey, sigalg, why_bogus,
7730                 LDNS_SECTION_ANSWER, NULL);
7731         if(sec == sec_status_secure) {
7732                 return 1;
7733         }
7734         if(why_bogus)
7735                 auth_zone_log(z->name, VERB_ALGO, "DNSSEC verify was bogus: %s", *why_bogus);
7736         return 0;
7737 }
7738
7739 /** check for nsec3, the RR with params equal, if bitmap has the type */
7740 static int nsec3_of_param_has_type(struct auth_rrset* nsec3, int algo,
7741         size_t iter, uint8_t* salt, size_t saltlen, uint16_t rrtype)
7742 {
7743         int i, count = (int)nsec3->data->count;
7744         struct ub_packed_rrset_key pk;
7745         memset(&pk, 0, sizeof(pk));
7746         pk.entry.data = nsec3->data;
7747         for(i=0; i<count; i++) {
7748                 int rralgo;
7749                 size_t rriter, rrsaltlen;
7750                 uint8_t* rrsalt;
7751                 if(!nsec3_get_params(&pk, i, &rralgo, &rriter, &rrsalt,
7752                         &rrsaltlen))
7753                         continue; /* no parameters, malformed */
7754                 if(rralgo != algo || rriter != iter || rrsaltlen != saltlen)
7755                         continue; /* different parameters */
7756                 if(saltlen != 0) {
7757                         if(rrsalt == NULL || salt == NULL)
7758                                 continue;
7759                         if(memcmp(rrsalt, salt, saltlen) != 0)
7760                                 continue; /* different salt parameters */
7761                 }
7762                 if(nsec3_has_type(&pk, i, rrtype))
7763                         return 1;
7764         }
7765         return 0;
7766 }
7767
7768 /** Verify the absence of ZONEMD with DNSSEC by checking NSEC, NSEC3 type flag.
7769  * return false on failure, reason contains description of failure. */
7770 static int zonemd_check_dnssec_absence(struct auth_zone* z,
7771         struct module_env* env, struct module_stack* mods,
7772         struct ub_packed_rrset_key* dnskey, struct auth_data* apex,
7773         char** reason, char** why_bogus, uint8_t* sigalg)
7774 {
7775         struct auth_rrset* nsec = NULL;
7776         if(!apex) {
7777                 *reason = "zone has no apex domain but ZONEMD missing";
7778                 return 0;
7779         }
7780         nsec = az_domain_rrset(apex, LDNS_RR_TYPE_NSEC);
7781         if(nsec) {
7782                 struct ub_packed_rrset_key pk;
7783                 /* dnssec verify the NSEC */
7784                 if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex,
7785                         nsec, why_bogus, sigalg)) {
7786                         *reason = "DNSSEC verify failed for NSEC RRset";
7787                         return 0;
7788                 }
7789                 /* check type bitmap */
7790                 memset(&pk, 0, sizeof(pk));
7791                 pk.entry.data = nsec->data;
7792                 if(nsec_has_type(&pk, LDNS_RR_TYPE_ZONEMD)) {
7793                         *reason = "DNSSEC NSEC bitmap says type ZONEMD exists";
7794                         return 0;
7795                 }
7796                 auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC NSEC verification of absence of ZONEMD secure");
7797         } else {
7798                 /* NSEC3 perhaps ? */
7799                 int algo;
7800                 size_t iter, saltlen;
7801                 uint8_t* salt;
7802                 struct auth_rrset* nsec3param = az_domain_rrset(apex,
7803                         LDNS_RR_TYPE_NSEC3PARAM);
7804                 struct auth_data* match;
7805                 struct auth_rrset* nsec3;
7806                 if(!nsec3param) {
7807                         *reason = "zone has no NSEC information but ZONEMD missing";
7808                         return 0;
7809                 }
7810                 if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen)) {
7811                         *reason = "zone has no NSEC information but ZONEMD missing";
7812                         return 0;
7813                 }
7814                 /* find the NSEC3 record */
7815                 match = az_nsec3_find_exact(z, z->name, z->namelen, algo,
7816                         iter, salt, saltlen);
7817                 if(!match) {
7818                         *reason = "zone has no NSEC3 domain for the apex but ZONEMD missing";
7819                         return 0;
7820                 }
7821                 nsec3 = az_domain_rrset(match, LDNS_RR_TYPE_NSEC3);
7822                 if(!nsec3) {
7823                         *reason = "zone has no NSEC3 RRset for the apex but ZONEMD missing";
7824                         return 0;
7825                 }
7826                 /* dnssec verify the NSEC3 */
7827                 if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, match,
7828                         nsec3, why_bogus, sigalg)) {
7829                         *reason = "DNSSEC verify failed for NSEC3 RRset";
7830                         return 0;
7831                 }
7832                 /* check type bitmap */
7833                 if(nsec3_of_param_has_type(nsec3, algo, iter, salt, saltlen,
7834                         LDNS_RR_TYPE_ZONEMD)) {
7835                         *reason = "DNSSEC NSEC3 bitmap says type ZONEMD exists";
7836                         return 0;
7837                 }
7838                 auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC NSEC3 verification of absence of ZONEMD secure");
7839         }
7840
7841         return 1;
7842 }
7843
7844 /** Verify the SOA and ZONEMD DNSSEC signatures.
7845  * return false on failure, reason contains description of failure. */
7846 static int zonemd_check_dnssec_soazonemd(struct auth_zone* z,
7847         struct module_env* env, struct module_stack* mods,
7848         struct ub_packed_rrset_key* dnskey, struct auth_data* apex,
7849         struct auth_rrset* zonemd_rrset, char** reason, char** why_bogus,
7850         uint8_t* sigalg)
7851 {
7852         struct auth_rrset* soa;
7853         if(!apex) {
7854                 *reason = "zone has no apex domain";
7855                 return 0;
7856         }
7857         soa = az_domain_rrset(apex, LDNS_RR_TYPE_SOA);
7858         if(!soa) {
7859                 *reason = "zone has no SOA RRset";
7860                 return 0;
7861         }
7862         if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex, soa,
7863                 why_bogus, sigalg)) {
7864                 *reason = "DNSSEC verify failed for SOA RRset";
7865                 return 0;
7866         }
7867         if(!zonemd_dnssec_verify_rrset(z, env, mods, dnskey, apex,
7868                 zonemd_rrset, why_bogus, sigalg)) {
7869                 *reason = "DNSSEC verify failed for ZONEMD RRset";
7870                 return 0;
7871         }
7872         auth_zone_log(z->name, VERB_ALGO, "zonemd DNSSEC verification of SOA and ZONEMD RRsets secure");
7873         return 1;
7874 }
7875
7876 /**
7877  * Fail the ZONEMD verification.
7878  * @param z: auth zone that fails.
7879  * @param env: environment with config, to ignore failure or not.
7880  * @param reason: failure string description.
7881  * @param why_bogus: failure string for DNSSEC verification failure.
7882  * @param result: strdup result in here if not NULL.
7883  */
7884 static void auth_zone_zonemd_fail(struct auth_zone* z, struct module_env* env,
7885         char* reason, char* why_bogus, char** result)
7886 {
7887         char zstr[255+1];
7888         /* if fail: log reason, and depending on config also take action
7889          * and drop the zone, eg. it is gone from memory, set zone_expired */
7890         dname_str(z->name, zstr);
7891         if(!reason) reason = "verification failed";
7892         if(result) {
7893                 if(why_bogus) {
7894                         char res[1024];
7895                         snprintf(res, sizeof(res), "%s: %s", reason,
7896                                 why_bogus);
7897                         *result = strdup(res);
7898                 } else {
7899                         *result = strdup(reason);
7900                 }
7901                 if(!*result) log_err("out of memory");
7902         } else {
7903                 log_warn("auth zone %s: ZONEMD verification failed: %s", zstr, reason);
7904         }
7905
7906         if(env->cfg->zonemd_permissive_mode) {
7907                 verbose(VERB_ALGO, "zonemd-permissive-mode enabled, "
7908                         "not blocking zone %s", zstr);
7909                 return;
7910         }
7911
7912         /* expired means the zone gives servfail and is not used by
7913          * lookup if fallback_enabled*/
7914         z->zone_expired = 1;
7915 }
7916
7917 /**
7918  * Verify the zonemd with DNSSEC and hash check, with given key.
7919  * @param z: auth zone.
7920  * @param env: environment with config and temp buffers.
7921  * @param mods: module stack with validator env for verification.
7922  * @param dnskey: dnskey that we can use, or NULL.  If nonnull, the key
7923  *      has been verified and is the start of the chain of trust.
7924  * @param is_insecure: if true, the dnskey is not used, the zone is insecure.
7925  *      And dnssec is not used.  It is DNSSEC secure insecure or not under
7926  *      a trust anchor.
7927  * @param sigalg: if nonNULL provide algorithm downgrade protection.
7928  *      Otherwise one algorithm is enough. Must have space of ALGO_NEEDS_MAX+1.
7929  * @param result: if not NULL result reason copied here.
7930  */
7931 static void
7932 auth_zone_verify_zonemd_with_key(struct auth_zone* z, struct module_env* env,
7933         struct module_stack* mods, struct ub_packed_rrset_key* dnskey,
7934         int is_insecure, char** result, uint8_t* sigalg)
7935 {
7936         char* reason = NULL, *why_bogus = NULL;
7937         struct auth_data* apex = NULL;
7938         struct auth_rrset* zonemd_rrset = NULL;
7939         int zonemd_absent = 0, zonemd_absence_dnssecok = 0;
7940
7941         /* see if ZONEMD is present or absent. */
7942         apex = az_find_name(z, z->name, z->namelen);
7943         if(!apex) {
7944                 zonemd_absent = 1;
7945         } else {
7946                 zonemd_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_ZONEMD);
7947                 if(!zonemd_rrset || zonemd_rrset->data->count==0) {
7948                         zonemd_absent = 1;
7949                         zonemd_rrset = NULL;
7950                 }
7951         }
7952
7953         /* if no DNSSEC, done. */
7954         /* if no ZONEMD, and DNSSEC, use DNSKEY to verify NSEC or NSEC3 for
7955          * zone apex.  Check ZONEMD bit is turned off or else fail */
7956         /* if ZONEMD, and DNSSEC, check DNSSEC signature on SOA and ZONEMD,
7957          * or else fail */
7958         if(!dnskey && !is_insecure) {
7959                 auth_zone_zonemd_fail(z, env, "DNSKEY missing", NULL, result);
7960                 return;
7961         } else if(!zonemd_rrset && dnskey && !is_insecure) {
7962                 /* fetch, DNSSEC verify, and check NSEC/NSEC3 */
7963                 if(!zonemd_check_dnssec_absence(z, env, mods, dnskey, apex,
7964                         &reason, &why_bogus, sigalg)) {
7965                         auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
7966                         return;
7967                 }
7968                 zonemd_absence_dnssecok = 1;
7969         } else if(zonemd_rrset && dnskey && !is_insecure) {
7970                 /* check DNSSEC verify of SOA and ZONEMD */
7971                 if(!zonemd_check_dnssec_soazonemd(z, env, mods, dnskey, apex,
7972                         zonemd_rrset, &reason, &why_bogus, sigalg)) {
7973                         auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
7974                         return;
7975                 }
7976         }
7977
7978         if(zonemd_absent && z->zonemd_reject_absence) {
7979                 auth_zone_zonemd_fail(z, env, "ZONEMD absent and that is not allowed by config", NULL, result);
7980                 return;
7981         }
7982         if(zonemd_absent && zonemd_absence_dnssecok) {
7983                 auth_zone_log(z->name, VERB_ALGO, "DNSSEC verified nonexistence of ZONEMD");
7984                 if(result) {
7985                         *result = strdup("DNSSEC verified nonexistence of ZONEMD");
7986                         if(!*result) log_err("out of memory");
7987                 }
7988                 return;
7989         }
7990         if(zonemd_absent) {
7991                 auth_zone_log(z->name, VERB_ALGO, "no ZONEMD present");
7992                 if(result) {
7993                         *result = strdup("no ZONEMD present");
7994                         if(!*result) log_err("out of memory");
7995                 }
7996                 return;
7997         }
7998
7999         /* check ZONEMD checksum and report or else fail. */
8000         if(!auth_zone_zonemd_check_hash(z, env, &reason)) {
8001                 auth_zone_zonemd_fail(z, env, reason, NULL, result);
8002                 return;
8003         }
8004
8005         /* success! log the success */
8006         auth_zone_log(z->name, VERB_ALGO, "ZONEMD verification successful");
8007         if(result) {
8008                 *result = strdup("ZONEMD verification successful");
8009                 if(!*result) log_err("out of memory");
8010         }
8011 }
8012
8013 /**
8014  * verify the zone DNSKEY rrset from the trust anchor
8015  * This is possible because the anchor is for the zone itself, and can
8016  * thus apply straight to the zone DNSKEY set.
8017  * @param z: the auth zone.
8018  * @param env: environment with time and temp buffers.
8019  * @param mods: module stack for validator environment for dnssec validation.
8020  * @param anchor: trust anchor to use
8021  * @param is_insecure: returned, true if the zone is securely insecure.
8022  * @param why_bogus: if the routine fails, returns the failure reason.
8023  * @param keystorage: where to store the ub_packed_rrset_key that is created
8024  *      on success. A pointer to it is returned on success.
8025  * @return the dnskey RRset, reference to zone data and keystorage, or
8026  *      NULL on failure.
8027  */
8028 static struct ub_packed_rrset_key*
8029 zonemd_get_dnskey_from_anchor(struct auth_zone* z, struct module_env* env,
8030         struct module_stack* mods, struct trust_anchor* anchor,
8031         int* is_insecure, char** why_bogus,
8032         struct ub_packed_rrset_key* keystorage)
8033 {
8034         struct auth_data* apex;
8035         struct auth_rrset* dnskey_rrset;
8036         enum sec_status sec;
8037         struct val_env* ve;
8038         int m;
8039
8040         apex = az_find_name(z, z->name, z->namelen);
8041         if(!apex) {
8042                 *why_bogus = "have trust anchor, but zone has no apex domain for DNSKEY";
8043                 return 0;
8044         }
8045         dnskey_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_DNSKEY);
8046         if(!dnskey_rrset || dnskey_rrset->data->count==0) {
8047                 *why_bogus = "have trust anchor, but zone has no DNSKEY";
8048                 return 0;
8049         }
8050
8051         m = modstack_find(mods, "validator");
8052         if(m == -1) {
8053                 *why_bogus = "have trust anchor, but no validator module";
8054                 return 0;
8055         }
8056         ve = (struct val_env*)env->modinfo[m];
8057
8058         memset(keystorage, 0, sizeof(*keystorage));
8059         keystorage->entry.key = keystorage;
8060         keystorage->entry.data = dnskey_rrset->data;
8061         keystorage->rk.dname = apex->name;
8062         keystorage->rk.dname_len = apex->namelen;
8063         keystorage->rk.type = htons(LDNS_RR_TYPE_DNSKEY);
8064         keystorage->rk.rrset_class = htons(z->dclass);
8065         auth_zone_log(z->name, VERB_QUERY,
8066                 "zonemd: verify DNSKEY RRset with trust anchor");
8067         sec = val_verify_DNSKEY_with_TA(env, ve, keystorage, anchor->ds_rrset,
8068                 anchor->dnskey_rrset, NULL, why_bogus, NULL);
8069         regional_free_all(env->scratch);
8070         if(sec == sec_status_secure) {
8071                 /* success */
8072                 *is_insecure = 0;
8073                 return keystorage;
8074         } else if(sec == sec_status_insecure) {
8075                 /* insecure */
8076                 *is_insecure = 1;
8077         } else {
8078                 /* bogus */
8079                 *is_insecure = 0;
8080                 auth_zone_log(z->name, VERB_ALGO,
8081                         "zonemd: verify DNSKEY RRset with trust anchor failed: %s", *why_bogus);
8082         }
8083         return NULL;
8084 }
8085
8086 /** verify the DNSKEY from the zone with looked up DS record */
8087 static struct ub_packed_rrset_key*
8088 auth_zone_verify_zonemd_key_with_ds(struct auth_zone* z,
8089         struct module_env* env, struct module_stack* mods,
8090         struct ub_packed_rrset_key* ds, int* is_insecure, char** why_bogus,
8091         struct ub_packed_rrset_key* keystorage, uint8_t* sigalg)
8092 {
8093         struct auth_data* apex;
8094         struct auth_rrset* dnskey_rrset;
8095         enum sec_status sec;
8096         struct val_env* ve;
8097         int m;
8098
8099         /* fetch DNSKEY from zone data */
8100         apex = az_find_name(z, z->name, z->namelen);
8101         if(!apex) {
8102                 *why_bogus = "in verifywithDS, zone has no apex";
8103                 return NULL;
8104         }
8105         dnskey_rrset = az_domain_rrset(apex, LDNS_RR_TYPE_DNSKEY);
8106         if(!dnskey_rrset || dnskey_rrset->data->count==0) {
8107                 *why_bogus = "in verifywithDS, zone has no DNSKEY";
8108                 return NULL;
8109         }
8110
8111         m = modstack_find(mods, "validator");
8112         if(m == -1) {
8113                 *why_bogus = "in verifywithDS, have no validator module";
8114                 return NULL;
8115         }
8116         ve = (struct val_env*)env->modinfo[m];
8117
8118         memset(keystorage, 0, sizeof(*keystorage));
8119         keystorage->entry.key = keystorage;
8120         keystorage->entry.data = dnskey_rrset->data;
8121         keystorage->rk.dname = apex->name;
8122         keystorage->rk.dname_len = apex->namelen;
8123         keystorage->rk.type = htons(LDNS_RR_TYPE_DNSKEY);
8124         keystorage->rk.rrset_class = htons(z->dclass);
8125         auth_zone_log(z->name, VERB_QUERY, "zonemd: verify zone DNSKEY with DS");
8126         sec = val_verify_DNSKEY_with_DS(env, ve, keystorage, ds, sigalg,
8127                 why_bogus, NULL);
8128         regional_free_all(env->scratch);
8129         if(sec == sec_status_secure) {
8130                 /* success */
8131                 return keystorage;
8132         } else if(sec == sec_status_insecure) {
8133                 /* insecure */
8134                 *is_insecure = 1;
8135         } else {
8136                 /* bogus */
8137                 *is_insecure = 0;
8138                 if(*why_bogus == NULL)
8139                         *why_bogus = "verify failed";
8140                 auth_zone_log(z->name, VERB_ALGO,
8141                         "zonemd: verify DNSKEY RRset with DS failed: %s",
8142                         *why_bogus);
8143         }
8144         return NULL;
8145 }
8146
8147 /** callback for ZONEMD lookup of DNSKEY */
8148 void auth_zonemd_dnskey_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
8149         enum sec_status sec, char* why_bogus, int ATTR_UNUSED(was_ratelimited))
8150 {
8151         struct auth_zone* z = (struct auth_zone*)arg;
8152         struct module_env* env;
8153         char* reason = NULL, *ds_bogus = NULL, *typestr="DNSKEY";
8154         struct ub_packed_rrset_key* dnskey = NULL, *ds = NULL;
8155         int is_insecure = 0, downprot;
8156         struct ub_packed_rrset_key keystorage;
8157         uint8_t sigalg[ALGO_NEEDS_MAX+1];
8158
8159         lock_rw_wrlock(&z->lock);
8160         env = z->zonemd_callback_env;
8161         /* release the env variable so another worker can pick up the
8162          * ZONEMD verification task if it wants to */
8163         z->zonemd_callback_env = NULL;
8164         if(!env || env->outnet->want_to_quit || z->zone_deleted) {
8165                 lock_rw_unlock(&z->lock);
8166                 return; /* stop on quit */
8167         }
8168         if(z->zonemd_callback_qtype == LDNS_RR_TYPE_DS)
8169                 typestr = "DS";
8170         downprot = env->cfg->harden_algo_downgrade;
8171
8172         /* process result */
8173         if(sec == sec_status_bogus) {
8174                 reason = why_bogus;
8175                 if(!reason) {
8176                         if(z->zonemd_callback_qtype == LDNS_RR_TYPE_DNSKEY)
8177                                 reason = "lookup of DNSKEY was bogus";
8178                         else    reason = "lookup of DS was bogus";
8179                 }
8180                 auth_zone_log(z->name, VERB_ALGO,
8181                         "zonemd lookup of %s was bogus: %s", typestr, reason);
8182         } else if(rcode == LDNS_RCODE_NOERROR) {
8183                 uint16_t wanted_qtype = z->zonemd_callback_qtype;
8184                 struct regional* temp = env->scratch;
8185                 struct query_info rq;
8186                 struct reply_info* rep;
8187                 memset(&rq, 0, sizeof(rq));
8188                 rep = parse_reply_in_temp_region(buf, temp, &rq);
8189                 if(rep && rq.qtype == wanted_qtype &&
8190                         query_dname_compare(z->name, rq.qname) == 0 &&
8191                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR) {
8192                         /* parsed successfully */
8193                         struct ub_packed_rrset_key* answer =
8194                                 reply_find_answer_rrset(&rq, rep);
8195                         if(answer && sec == sec_status_secure) {
8196                                 if(z->zonemd_callback_qtype == LDNS_RR_TYPE_DNSKEY)
8197                                         dnskey = answer;
8198                                 else    ds = answer;
8199                                 auth_zone_log(z->name, VERB_ALGO,
8200                                         "zonemd lookup of %s was secure", typestr);
8201                         } else if(sec == sec_status_secure && !answer) {
8202                                 is_insecure = 1;
8203                                 auth_zone_log(z->name, VERB_ALGO,
8204                                         "zonemd lookup of %s has no content, but is secure, treat as insecure", typestr);
8205                         } else if(sec == sec_status_insecure) {
8206                                 is_insecure = 1;
8207                                 auth_zone_log(z->name, VERB_ALGO,
8208                                         "zonemd lookup of %s was insecure", typestr);
8209                         } else if(sec == sec_status_indeterminate) {
8210                                 is_insecure = 1;
8211                                 auth_zone_log(z->name, VERB_ALGO,
8212                                         "zonemd lookup of %s was indeterminate, treat as insecure", typestr);
8213                         } else {
8214                                 auth_zone_log(z->name, VERB_ALGO,
8215                                         "zonemd lookup of %s has nodata", typestr);
8216                                 if(z->zonemd_callback_qtype == LDNS_RR_TYPE_DNSKEY)
8217                                         reason = "lookup of DNSKEY has nodata";
8218                                 else    reason = "lookup of DS has nodata";
8219                         }
8220                 } else if(rep && rq.qtype == wanted_qtype &&
8221                         query_dname_compare(z->name, rq.qname) == 0 &&
8222                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8223                         sec == sec_status_secure) {
8224                         /* secure nxdomain, so the zone is like some RPZ zone
8225                          * that does not exist in the wider internet, with
8226                          * a secure nxdomain answer outside of it. So we
8227                          * treat the zonemd zone without a dnssec chain of
8228                          * trust, as insecure. */
8229                         is_insecure = 1;
8230                         auth_zone_log(z->name, VERB_ALGO,
8231                                 "zonemd lookup of %s was secure NXDOMAIN, treat as insecure", typestr);
8232                 } else if(rep && rq.qtype == wanted_qtype &&
8233                         query_dname_compare(z->name, rq.qname) == 0 &&
8234                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8235                         sec == sec_status_insecure) {
8236                         is_insecure = 1;
8237                         auth_zone_log(z->name, VERB_ALGO,
8238                                 "zonemd lookup of %s was insecure NXDOMAIN, treat as insecure", typestr);
8239                 } else if(rep && rq.qtype == wanted_qtype &&
8240                         query_dname_compare(z->name, rq.qname) == 0 &&
8241                         FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN &&
8242                         sec == sec_status_indeterminate) {
8243                         is_insecure = 1;
8244                         auth_zone_log(z->name, VERB_ALGO,
8245                                 "zonemd lookup of %s was indeterminate NXDOMAIN, treat as insecure", typestr);
8246                 } else {
8247                         auth_zone_log(z->name, VERB_ALGO,
8248                                 "zonemd lookup of %s has no answer", typestr);
8249                         if(z->zonemd_callback_qtype == LDNS_RR_TYPE_DNSKEY)
8250                                 reason = "lookup of DNSKEY has no answer";
8251                         else    reason = "lookup of DS has no answer";
8252                 }
8253         } else {
8254                 auth_zone_log(z->name, VERB_ALGO,
8255                         "zonemd lookup of %s failed", typestr);
8256                 if(z->zonemd_callback_qtype == LDNS_RR_TYPE_DNSKEY)
8257                         reason = "lookup of DNSKEY failed";
8258                 else    reason = "lookup of DS failed";
8259         }
8260
8261         if(!reason && !is_insecure && !dnskey && ds) {
8262                 dnskey = auth_zone_verify_zonemd_key_with_ds(z, env,
8263                         &env->mesh->mods, ds, &is_insecure, &ds_bogus,
8264                         &keystorage, downprot?sigalg:NULL);
8265                 if(!dnskey && !is_insecure && !reason)
8266                         reason = "DNSKEY verify with DS failed";
8267         }
8268
8269         if(reason) {
8270                 auth_zone_zonemd_fail(z, env, reason, ds_bogus, NULL);
8271                 lock_rw_unlock(&z->lock);
8272                 return;
8273         }
8274
8275         auth_zone_verify_zonemd_with_key(z, env, &env->mesh->mods, dnskey,
8276                 is_insecure, NULL, downprot?sigalg:NULL);
8277         regional_free_all(env->scratch);
8278         lock_rw_unlock(&z->lock);
8279 }
8280
8281 /** lookup DNSKEY for ZONEMD verification */
8282 static int
8283 zonemd_lookup_dnskey(struct auth_zone* z, struct module_env* env)
8284 {
8285         struct query_info qinfo;
8286         uint16_t qflags = BIT_RD;
8287         struct edns_data edns;
8288         sldns_buffer* buf = env->scratch_buffer;
8289         int fetch_ds = 0;
8290
8291         if(!z->fallback_enabled) {
8292                 /* we cannot actually get the DNSKEY, because it is in the
8293                  * zone we have ourselves, and it is not served yet
8294                  * (possibly), so fetch type DS */
8295                 fetch_ds = 1;
8296         }
8297         if(z->zonemd_callback_env) {
8298                 /* another worker is already working on the callback
8299                  * for the DNSKEY lookup for ZONEMD verification.
8300                  * We do not also have to do ZONEMD verification, let that
8301                  * worker do it */
8302                 auth_zone_log(z->name, VERB_ALGO,
8303                         "zonemd needs lookup of %s and that already is worked on by another worker", (fetch_ds?"DS":"DNSKEY"));
8304                 return 1;
8305         }
8306
8307         /* use mesh_new_callback to lookup the DNSKEY,
8308          * and then wait for them to be looked up (in cache, or query) */
8309         qinfo.qname_len = z->namelen;
8310         qinfo.qname = z->name;
8311         qinfo.qclass = z->dclass;
8312         if(fetch_ds)
8313                 qinfo.qtype = LDNS_RR_TYPE_DS;
8314         else    qinfo.qtype = LDNS_RR_TYPE_DNSKEY;
8315         qinfo.local_alias = NULL;
8316         if(verbosity >= VERB_ALGO) {
8317                 char buf1[512];
8318                 char buf2[LDNS_MAX_DOMAINLEN+1];
8319                 dname_str(z->name, buf2);
8320                 snprintf(buf1, sizeof(buf1), "auth zone %s: lookup %s "
8321                         "for zonemd verification", buf2,
8322                         (fetch_ds?"DS":"DNSKEY"));
8323                 log_query_info(VERB_ALGO, buf1, &qinfo);
8324         }
8325         edns.edns_present = 1;
8326         edns.ext_rcode = 0;
8327         edns.edns_version = 0;
8328         edns.bits = EDNS_DO;
8329         edns.opt_list_in = NULL;
8330         edns.opt_list_out = NULL;
8331         edns.opt_list_inplace_cb_out = NULL;
8332         if(sldns_buffer_capacity(buf) < 65535)
8333                 edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
8334         else    edns.udp_size = 65535;
8335
8336         /* store the worker-specific module env for the callback.
8337          * We can then reference this when the callback executes */
8338         z->zonemd_callback_env = env;
8339         z->zonemd_callback_qtype = qinfo.qtype;
8340         /* the callback can be called straight away */
8341         lock_rw_unlock(&z->lock);
8342         if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
8343                 &auth_zonemd_dnskey_lookup_callback, z)) {
8344                 lock_rw_wrlock(&z->lock);
8345                 log_err("out of memory lookup of %s for zonemd",
8346                         (fetch_ds?"DS":"DNSKEY"));
8347                 return 0;
8348         }
8349         lock_rw_wrlock(&z->lock);
8350         return 1;
8351 }
8352
8353 void auth_zone_verify_zonemd(struct auth_zone* z, struct module_env* env,
8354         struct module_stack* mods, char** result, int offline, int only_online)
8355 {
8356         char* reason = NULL, *why_bogus = NULL;
8357         struct trust_anchor* anchor = NULL;
8358         struct ub_packed_rrset_key* dnskey = NULL;
8359         struct ub_packed_rrset_key keystorage;
8360         int is_insecure = 0;
8361         /* verify the ZONEMD if present.
8362          * If not present check if absence is allowed by DNSSEC */
8363         if(!z->zonemd_check)
8364                 return;
8365         if(z->data.count == 0)
8366                 return; /* no data */
8367
8368         /* if zone is under a trustanchor */
8369         /* is it equal to trustanchor - get dnskey's verified */
8370         /* else, find chain of trust by fetching DNSKEYs lookup for zone */
8371         /* result if that, if insecure, means no DNSSEC for the ZONEMD,
8372          * otherwise we have the zone DNSKEY for the DNSSEC verification. */
8373         if(env->anchors)
8374                 anchor = anchors_lookup(env->anchors, z->name, z->namelen,
8375                         z->dclass);
8376         if(anchor && anchor->numDS == 0 && anchor->numDNSKEY == 0) {
8377                 /* domain-insecure trust anchor for unsigned zones */
8378                 lock_basic_unlock(&anchor->lock);
8379                 if(only_online)
8380                         return;
8381                 dnskey = NULL;
8382                 is_insecure = 1;
8383         } else if(anchor && query_dname_compare(z->name, anchor->name) == 0) {
8384                 if(only_online) {
8385                         lock_basic_unlock(&anchor->lock);
8386                         return;
8387                 }
8388                 /* equal to trustanchor, no need for online lookups */
8389                 dnskey = zonemd_get_dnskey_from_anchor(z, env, mods, anchor,
8390                         &is_insecure, &why_bogus, &keystorage);
8391                 lock_basic_unlock(&anchor->lock);
8392                 if(!dnskey && !reason && !is_insecure) {
8393                         reason = "verify DNSKEY RRset with trust anchor failed";
8394                 }
8395         } else if(anchor) {
8396                 lock_basic_unlock(&anchor->lock);
8397                 /* perform online lookups */
8398                 if(offline)
8399                         return;
8400                 /* setup online lookups, and wait for them */
8401                 if(zonemd_lookup_dnskey(z, env)) {
8402                         /* wait for the lookup */
8403                         return;
8404                 }
8405                 reason = "could not lookup DNSKEY for chain of trust";
8406         } else {
8407                 /* the zone is not under a trust anchor */
8408                 if(only_online)
8409                         return;
8410                 dnskey = NULL;
8411                 is_insecure = 1;
8412         }
8413
8414         if(reason) {
8415                 auth_zone_zonemd_fail(z, env, reason, why_bogus, result);
8416                 return;
8417         }
8418
8419         auth_zone_verify_zonemd_with_key(z, env, mods, dnskey, is_insecure,
8420                 result, NULL);
8421         regional_free_all(env->scratch);
8422 }
8423
8424 void auth_zones_pickup_zonemd_verify(struct auth_zones* az,
8425         struct module_env* env)
8426 {
8427         struct auth_zone key;
8428         uint8_t savezname[255+1];
8429         size_t savezname_len;
8430         struct auth_zone* z;
8431         key.node.key = &key;
8432         lock_rw_rdlock(&az->lock);
8433         RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
8434                 lock_rw_wrlock(&z->lock);
8435                 if(!z->zonemd_check) {
8436                         lock_rw_unlock(&z->lock);
8437                         continue;
8438                 }
8439                 key.dclass = z->dclass;
8440                 key.namelabs = z->namelabs;
8441                 if(z->namelen > sizeof(savezname)) {
8442                         lock_rw_unlock(&z->lock);
8443                         log_err("auth_zones_pickup_zonemd_verify: zone name too long");
8444                         continue;
8445                 }
8446                 savezname_len = z->namelen;
8447                 memmove(savezname, z->name, z->namelen);
8448                 lock_rw_unlock(&az->lock);
8449                 auth_zone_verify_zonemd(z, env, &env->mesh->mods, NULL, 0, 1);
8450                 lock_rw_unlock(&z->lock);
8451                 lock_rw_rdlock(&az->lock);
8452                 /* find the zone we had before, it is not deleted,
8453                  * because we have a flag for that that is processed at
8454                  * apply_cfg time */
8455                 key.namelen = savezname_len;
8456                 key.name = savezname;
8457                 z = (struct auth_zone*)rbtree_search(&az->ztree, &key);
8458                 if(!z)
8459                         break;
8460         }
8461         lock_rw_unlock(&az->lock);
8462 }