contrib/unbound/validator/val_secalgo.c

   1 /*
   2  * validator/val_secalgo.c - validator security algorithm functions.
   3  *
   4  * Copyright (c) 2012, NLnet Labs. All rights reserved.
   5  *
   6  * This software is open source.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * Redistributions of source code must retain the above copyright notice,
  13  * this list of conditions and the following disclaimer.
  14  *
  15  * Redistributions in binary form must reproduce the above copyright notice,
  16  * this list of conditions and the following disclaimer in the documentation
  17  * and/or other materials provided with the distribution.
  18  *
  19  * Neither the name of the NLNET LABS nor the names of its contributors may
  20  * be used to endorse or promote products derived from this software without
  21  * specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  29  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  30  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  31  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  32  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  33  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /**
  37  * \file
  38  *
  39  * This file contains helper functions for the validator module.
  40  * These functions take raw data buffers, formatted for crypto verification,
  41  * and do the library calls (for the crypto library in use).
  42  */
  43 #include "config.h"
  44 /* packed_rrset on top to define enum types (forced by c99 standard) */
  45 #include "util/data/packed_rrset.h"
  46 #include "validator/val_secalgo.h"
  47 #include "validator/val_nsec3.h"
  48 #include "util/log.h"
  49 #include "sldns/rrdef.h"
  50 #include "sldns/keyraw.h"
  51 #include "sldns/sbuffer.h"
  52
  53 #if !defined(HAVE_SSL) && !defined(HAVE_NSS) && !defined(HAVE_NETTLE)
  54 #error "Need crypto library to do digital signature cryptography"
  55 #endif
  56
  57 /* OpenSSL implementation */
  58 #ifdef HAVE_SSL
  59 #ifdef HAVE_OPENSSL_ERR_H
  60 #include <openssl/err.h>
  61 #endif
  62
  63 #ifdef HAVE_OPENSSL_RAND_H
  64 #include <openssl/rand.h>
  65 #endif
  66
  67 #ifdef HAVE_OPENSSL_CONF_H
  68 #include <openssl/conf.h>
  69 #endif
  70
  71 #ifdef HAVE_OPENSSL_ENGINE_H
  72 #include <openssl/engine.h>
  73 #endif
  74
  75 /* return size of digest if supported, or 0 otherwise */
  76 size_t
  77 nsec3_hash_algo_size_supported(int id)
  78 {
  79         switch(id) {
  80         case NSEC3_HASH_SHA1:
  81                 return SHA_DIGEST_LENGTH;
  82         default:
  83                 return 0;
  84         }
  85 }
  86
  87 /* perform nsec3 hash. return false on failure */
  88 int
  89 secalgo_nsec3_hash(int algo, unsigned char* buf, size_t len,
  90         unsigned char* res)
  91 {
  92         switch(algo) {
  93         case NSEC3_HASH_SHA1:
  94                 (void)SHA1(buf, len, res);
  95                 return 1;
  96         default:
  97                 return 0;
  98         }
  99 }
 100
 101 void
 102 secalgo_hash_sha256(unsigned char* buf, size_t len, unsigned char* res)
 103 {
 104         (void)SHA256(buf, len, res);
 105 }
 106
 107 /**
 108  * Return size of DS digest according to its hash algorithm.
 109  * @param algo: DS digest algo.
 110  * @return size in bytes of digest, or 0 if not supported.
 111  */
 112 size_t
 113 ds_digest_size_supported(int algo)
 114 {
 115         switch(algo) {
 116 #ifdef HAVE_EVP_SHA1
 117                 case LDNS_SHA1:
 118                         return SHA_DIGEST_LENGTH;
 119 #endif
 120 #ifdef HAVE_EVP_SHA256
 121                 case LDNS_SHA256:
 122                         return SHA256_DIGEST_LENGTH;
 123 #endif
 124 #ifdef USE_GOST
 125                 case LDNS_HASH_GOST:
 126                         /* we support GOST if it can be loaded */
 127                         (void)sldns_key_EVP_load_gost_id();
 128                         if(EVP_get_digestbyname("md_gost94"))
 129                                 return 32;
 130                         else    return 0;
 131 #endif
 132 #ifdef USE_ECDSA
 133                 case LDNS_SHA384:
 134                         return SHA384_DIGEST_LENGTH;
 135 #endif
 136                 default: break;
 137         }
 138         return 0;
 139 }
 140
 141 #ifdef USE_GOST
 142 /** Perform GOST hash */
 143 static int
 144 do_gost94(unsigned char* data, size_t len, unsigned char* dest)
 145 {
 146         const EVP_MD* md = EVP_get_digestbyname("md_gost94");
 147         if(!md)
 148                 return 0;
 149         return sldns_digest_evp(data, (unsigned int)len, dest, md);
 150 }
 151 #endif
 152
 153 int
 154 secalgo_ds_digest(int algo, unsigned char* buf, size_t len,
 155         unsigned char* res)
 156 {
 157         switch(algo) {
 158 #ifdef HAVE_EVP_SHA1
 159                 case LDNS_SHA1:
 160                         (void)SHA1(buf, len, res);
 161                         return 1;
 162 #endif
 163 #ifdef HAVE_EVP_SHA256
 164                 case LDNS_SHA256:
 165                         (void)SHA256(buf, len, res);
 166                         return 1;
 167 #endif
 168 #ifdef USE_GOST
 169                 case LDNS_HASH_GOST:
 170                         if(do_gost94(buf, len, res))
 171                                 return 1;
 172                         break;
 173 #endif
 174 #ifdef USE_ECDSA
 175                 case LDNS_SHA384:
 176                         (void)SHA384(buf, len, res);
 177                         return 1;
 178 #endif
 179                 default:
 180                         verbose(VERB_QUERY, "unknown DS digest algorithm %d",
 181                                 algo);
 182                         break;
 183         }
 184         return 0;
 185 }
 186
 187 /** return true if DNSKEY algorithm id is supported */
 188 int
 189 dnskey_algo_id_is_supported(int id)
 190 {
 191         switch(id) {
 192         case LDNS_RSAMD5:
 193                 /* RFC 6725 deprecates RSAMD5 */
 194                 return 0;
 195 #ifdef USE_DSA
 196         case LDNS_DSA:
 197         case LDNS_DSA_NSEC3:
 198 #endif
 199         case LDNS_RSASHA1:
 200         case LDNS_RSASHA1_NSEC3:
 201 #if defined(HAVE_EVP_SHA256) && defined(USE_SHA2)
 202         case LDNS_RSASHA256:
 203 #endif
 204 #if defined(HAVE_EVP_SHA512) && defined(USE_SHA2)
 205         case LDNS_RSASHA512:
 206 #endif
 207 #ifdef USE_ECDSA
 208         case LDNS_ECDSAP256SHA256:
 209         case LDNS_ECDSAP384SHA384:
 210 #endif
 211                 return 1;
 212 #ifdef USE_GOST
 213         case LDNS_ECC_GOST:
 214                 /* we support GOST if it can be loaded */
 215                 return sldns_key_EVP_load_gost_id();
 216 #endif
 217         default:
 218                 return 0;
 219         }
 220 }
 221
 222 /**
 223  * Output a libcrypto openssl error to the logfile.
 224  * @param str: string to add to it.
 225  * @param e: the error to output, error number from ERR_get_error().
 226  */
 227 static void
 228 log_crypto_error(const char* str, unsigned long e)
 229 {
 230         char buf[128];
 231         /* or use ERR_error_string if ERR_error_string_n is not avail TODO */
 232         ERR_error_string_n(e, buf, sizeof(buf));
 233         /* buf now contains */
 234         /* error:[error code]:[library name]:[function name]:[reason string] */
 235         log_err("%s crypto %s", str, buf);
 236 }
 237
 238 #ifdef USE_DSA
 239 /**
 240  * Setup DSA key digest in DER encoding ...
 241  * @param sig: input is signature output alloced ptr (unless failure).
 242  *      caller must free alloced ptr if this routine returns true.
 243  * @param len: input is initial siglen, output is output len.
 244  * @return false on failure.
 245  */
 246 static int
 247 setup_dsa_sig(unsigned char** sig, unsigned int* len)
 248 {
 249         unsigned char* orig = *sig;
 250         unsigned int origlen = *len;
 251         int newlen;
 252         BIGNUM *R, *S;
 253         DSA_SIG *dsasig;
 254
 255         /* extract the R and S field from the sig buffer */
 256         if(origlen < 1 + 2*SHA_DIGEST_LENGTH)
 257                 return 0;
 258         R = BN_new();
 259         if(!R) return 0;
 260         (void) BN_bin2bn(orig + 1, SHA_DIGEST_LENGTH, R);
 261         S = BN_new();
 262         if(!S) return 0;
 263         (void) BN_bin2bn(orig + 21, SHA_DIGEST_LENGTH, S);
 264         dsasig = DSA_SIG_new();
 265         if(!dsasig) return 0;
 266
 267         dsasig->r = R;
 268         dsasig->s = S;
 269         *sig = NULL;
 270         newlen = i2d_DSA_SIG(dsasig, sig);
 271         if(newlen < 0) {
 272                 DSA_SIG_free(dsasig);
 273                 free(*sig);
 274                 return 0;
 275         }
 276         *len = (unsigned int)newlen;
 277         DSA_SIG_free(dsasig);
 278         return 1;
 279 }
 280 #endif /* USE_DSA */
 281
 282 #ifdef USE_ECDSA
 283 /**
 284  * Setup the ECDSA signature in its encoding that the library wants.
 285  * Converts from plain numbers to ASN formatted.
 286  * @param sig: input is signature, output alloced ptr (unless failure).
 287  *      caller must free alloced ptr if this routine returns true.
 288  * @param len: input is initial siglen, output is output len.
 289  * @return false on failure.
 290  */
 291 static int
 292 setup_ecdsa_sig(unsigned char** sig, unsigned int* len)
 293 {
 294         /* convert from two BIGNUMs in the rdata buffer, to ASN notation.
 295          * ASN preable:  30440220 <R 32bytefor256> 0220 <S 32bytefor256>
 296          * the '20' is the length of that field (=bnsize).
 297 i        * the '44' is the total remaining length.
 298          * if negative, start with leading zero.
 299          * if starts with 00s, remove them from the number.
 300          */
 301         uint8_t pre[] = {0x30, 0x44, 0x02, 0x20};
 302         int pre_len = 4;
 303         uint8_t mid[] = {0x02, 0x20};
 304         int mid_len = 2;
 305         int raw_sig_len, r_high, s_high, r_rem=0, s_rem=0;
 306         int bnsize = (int)((*len)/2);
 307         unsigned char* d = *sig;
 308         uint8_t* p;
 309         /* if too short or not even length, fails */
 310         if(*len < 16 || bnsize*2 != (int)*len)
 311                 return 0;
 312
 313         /* strip leading zeroes from r (but not last one) */
 314         while(r_rem < bnsize-1 && d[r_rem] == 0)
 315                 r_rem++;
 316         /* strip leading zeroes from s (but not last one) */
 317         while(s_rem < bnsize-1 && d[bnsize+s_rem] == 0)
 318                 s_rem++;
 319
 320         r_high = ((d[0+r_rem]&0x80)?1:0);
 321         s_high = ((d[bnsize+s_rem]&0x80)?1:0);
 322         raw_sig_len = pre_len + r_high + bnsize - r_rem + mid_len +
 323                 s_high + bnsize - s_rem;
 324         *sig = (unsigned char*)malloc((size_t)raw_sig_len);
 325         if(!*sig)
 326                 return 0;
 327         p = (uint8_t*)*sig;
 328         p[0] = pre[0];
 329         p[1] = (uint8_t)(raw_sig_len-2);
 330         p[2] = pre[2];
 331         p[3] = (uint8_t)(bnsize + r_high - r_rem);
 332         p += 4;
 333         if(r_high) {
 334                 *p = 0;
 335                 p += 1;
 336         }
 337         memmove(p, d+r_rem, (size_t)bnsize-r_rem);
 338         p += bnsize-r_rem;
 339         memmove(p, mid, (size_t)mid_len-1);
 340         p += mid_len-1;
 341         *p = (uint8_t)(bnsize + s_high - s_rem);
 342         p += 1;
 343         if(s_high) {
 344                 *p = 0;
 345                 p += 1;
 346         }
 347         memmove(p, d+bnsize+s_rem, (size_t)bnsize-s_rem);
 348         *len = (unsigned int)raw_sig_len;
 349         return 1;
 350 }
 351 #endif /* USE_ECDSA */
 352
 353 #ifdef USE_ECDSA_EVP_WORKAROUND
 354 static EVP_MD ecdsa_evp_256_md;
 355 static EVP_MD ecdsa_evp_384_md;
 356 void ecdsa_evp_workaround_init(void)
 357 {
 358         /* openssl before 1.0.0 fixes RSA with the SHA256
 359          * hash in EVP.  We create one for ecdsa_sha256 */
 360         ecdsa_evp_256_md = *EVP_sha256();
 361         ecdsa_evp_256_md.required_pkey_type[0] = EVP_PKEY_EC;
 362         ecdsa_evp_256_md.verify = (void*)ECDSA_verify;
 363
 364         ecdsa_evp_384_md = *EVP_sha384();
 365         ecdsa_evp_384_md.required_pkey_type[0] = EVP_PKEY_EC;
 366         ecdsa_evp_384_md.verify = (void*)ECDSA_verify;
 367 }
 368 #endif /* USE_ECDSA_EVP_WORKAROUND */
 369
 370 /**
 371  * Setup key and digest for verification. Adjust sig if necessary.
 372  *
 373  * @param algo: key algorithm
 374  * @param evp_key: EVP PKEY public key to create.
 375  * @param digest_type: digest type to use
 376  * @param key: key to setup for.
 377  * @param keylen: length of key.
 378  * @return false on failure.
 379  */
 380 static int
 381 setup_key_digest(int algo, EVP_PKEY** evp_key, const EVP_MD** digest_type,
 382         unsigned char* key, size_t keylen)
 383 {
 384 #ifdef USE_DSA
 385         DSA* dsa;
 386 #endif
 387         RSA* rsa;
 388
 389         switch(algo) {
 390 #ifdef USE_DSA
 391                 case LDNS_DSA:
 392                 case LDNS_DSA_NSEC3:
 393                         *evp_key = EVP_PKEY_new();
 394                         if(!*evp_key) {
 395                                 log_err("verify: malloc failure in crypto");
 396                                 return 0;
 397                         }
 398                         dsa = sldns_key_buf2dsa_raw(key, keylen);
 399                         if(!dsa) {
 400                                 verbose(VERB_QUERY, "verify: "
 401                                         "sldns_key_buf2dsa_raw failed");
 402                                 return 0;
 403                         }
 404                         if(EVP_PKEY_assign_DSA(*evp_key, dsa) == 0) {
 405                                 verbose(VERB_QUERY, "verify: "
 406                                         "EVP_PKEY_assign_DSA failed");
 407                                 return 0;
 408                         }
 409                         *digest_type = EVP_dss1();
 410
 411                         break;
 412 #endif /* USE_DSA */
 413                 case LDNS_RSASHA1:
 414                 case LDNS_RSASHA1_NSEC3:
 415 #if defined(HAVE_EVP_SHA256) && defined(USE_SHA2)
 416                 case LDNS_RSASHA256:
 417 #endif
 418 #if defined(HAVE_EVP_SHA512) && defined(USE_SHA2)
 419                 case LDNS_RSASHA512:
 420 #endif
 421                         *evp_key = EVP_PKEY_new();
 422                         if(!*evp_key) {
 423                                 log_err("verify: malloc failure in crypto");
 424                                 return 0;
 425                         }
 426                         rsa = sldns_key_buf2rsa_raw(key, keylen);
 427                         if(!rsa) {
 428                                 verbose(VERB_QUERY, "verify: "
 429                                         "sldns_key_buf2rsa_raw SHA failed");
 430                                 return 0;
 431                         }
 432                         if(EVP_PKEY_assign_RSA(*evp_key, rsa) == 0) {
 433                                 verbose(VERB_QUERY, "verify: "
 434                                         "EVP_PKEY_assign_RSA SHA failed");
 435                                 return 0;
 436                         }
 437
 438                         /* select SHA version */
 439 #if defined(HAVE_EVP_SHA256) && defined(USE_SHA2)
 440                         if(algo == LDNS_RSASHA256)
 441                                 *digest_type = EVP_sha256();
 442                         else
 443 #endif
 444 #if defined(HAVE_EVP_SHA512) && defined(USE_SHA2)
 445                                 if(algo == LDNS_RSASHA512)
 446                                 *digest_type = EVP_sha512();
 447                         else
 448 #endif
 449                                 *digest_type = EVP_sha1();
 450
 451                         break;
 452                 case LDNS_RSAMD5:
 453                         *evp_key = EVP_PKEY_new();
 454                         if(!*evp_key) {
 455                                 log_err("verify: malloc failure in crypto");
 456                                 return 0;
 457                         }
 458                         rsa = sldns_key_buf2rsa_raw(key, keylen);
 459                         if(!rsa) {
 460                                 verbose(VERB_QUERY, "verify: "
 461                                         "sldns_key_buf2rsa_raw MD5 failed");
 462                                 return 0;
 463                         }
 464                         if(EVP_PKEY_assign_RSA(*evp_key, rsa) == 0) {
 465                                 verbose(VERB_QUERY, "verify: "
 466                                         "EVP_PKEY_assign_RSA MD5 failed");
 467                                 return 0;
 468                         }
 469                         *digest_type = EVP_md5();
 470
 471                         break;
 472 #ifdef USE_GOST
 473                 case LDNS_ECC_GOST:
 474                         *evp_key = sldns_gost2pkey_raw(key, keylen);
 475                         if(!*evp_key) {
 476                                 verbose(VERB_QUERY, "verify: "
 477                                         "sldns_gost2pkey_raw failed");
 478                                 return 0;
 479                         }
 480                         *digest_type = EVP_get_digestbyname("md_gost94");
 481                         if(!*digest_type) {
 482                                 verbose(VERB_QUERY, "verify: "
 483                                         "EVP_getdigest md_gost94 failed");
 484                                 return 0;
 485                         }
 486                         break;
 487 #endif
 488 #ifdef USE_ECDSA
 489                 case LDNS_ECDSAP256SHA256:
 490                         *evp_key = sldns_ecdsa2pkey_raw(key, keylen,
 491                                 LDNS_ECDSAP256SHA256);
 492                         if(!*evp_key) {
 493                                 verbose(VERB_QUERY, "verify: "
 494                                         "sldns_ecdsa2pkey_raw failed");
 495                                 return 0;
 496                         }
 497 #ifdef USE_ECDSA_EVP_WORKAROUND
 498                         *digest_type = &ecdsa_evp_256_md;
 499 #else
 500                         *digest_type = EVP_sha256();
 501 #endif
 502                         break;
 503                 case LDNS_ECDSAP384SHA384:
 504                         *evp_key = sldns_ecdsa2pkey_raw(key, keylen,
 505                                 LDNS_ECDSAP384SHA384);
 506                         if(!*evp_key) {
 507                                 verbose(VERB_QUERY, "verify: "
 508                                         "sldns_ecdsa2pkey_raw failed");
 509                                 return 0;
 510                         }
 511 #ifdef USE_ECDSA_EVP_WORKAROUND
 512                         *digest_type = &ecdsa_evp_384_md;
 513 #else
 514                         *digest_type = EVP_sha384();
 515 #endif
 516                         break;
 517 #endif /* USE_ECDSA */
 518                 default:
 519                         verbose(VERB_QUERY, "verify: unknown algorithm %d",
 520                                 algo);
 521                         return 0;
 522         }
 523         return 1;
 524 }
 525
 526 /**
 527  * Check a canonical sig+rrset and signature against a dnskey
 528  * @param buf: buffer with data to verify, the first rrsig part and the
 529  *      canonicalized rrset.
 530  * @param algo: DNSKEY algorithm.
 531  * @param sigblock: signature rdata field from RRSIG
 532  * @param sigblock_len: length of sigblock data.
 533  * @param key: public key data from DNSKEY RR.
 534  * @param keylen: length of keydata.
 535  * @param reason: bogus reason in more detail.
 536  * @return secure if verification succeeded, bogus on crypto failure,
 537  *      unchecked on format errors and alloc failures.
 538  */
 539 enum sec_status
 540 verify_canonrrset(sldns_buffer* buf, int algo, unsigned char* sigblock,
 541         unsigned int sigblock_len, unsigned char* key, unsigned int keylen,
 542         char** reason)
 543 {
 544         const EVP_MD *digest_type;
 545         EVP_MD_CTX* ctx;
 546         int res, dofree = 0, docrypto_free = 0;
 547         EVP_PKEY *evp_key = NULL;
 548
 549         if(!setup_key_digest(algo, &evp_key, &digest_type, key, keylen)) {
 550                 verbose(VERB_QUERY, "verify: failed to setup key");
 551                 *reason = "use of key for crypto failed";
 552                 EVP_PKEY_free(evp_key);
 553                 return sec_status_bogus;
 554         }
 555 #ifdef USE_DSA
 556         /* if it is a DSA signature in bind format, convert to DER format */
 557         if((algo == LDNS_DSA || algo == LDNS_DSA_NSEC3) &&
 558                 sigblock_len == 1+2*SHA_DIGEST_LENGTH) {
 559                 if(!setup_dsa_sig(&sigblock, &sigblock_len)) {
 560                         verbose(VERB_QUERY, "verify: failed to setup DSA sig");
 561                         *reason = "use of key for DSA crypto failed";
 562                         EVP_PKEY_free(evp_key);
 563                         return sec_status_bogus;
 564                 }
 565                 docrypto_free = 1;
 566         }
 567 #endif
 568 #if defined(USE_ECDSA) && defined(USE_DSA)
 569         else
 570 #endif
 571 #ifdef USE_ECDSA
 572         if(algo == LDNS_ECDSAP256SHA256 || algo == LDNS_ECDSAP384SHA384) {
 573                 /* EVP uses ASN prefix on sig, which is not in the wire data */
 574                 if(!setup_ecdsa_sig(&sigblock, &sigblock_len)) {
 575                         verbose(VERB_QUERY, "verify: failed to setup ECDSA sig");
 576                         *reason = "use of signature for ECDSA crypto failed";
 577                         EVP_PKEY_free(evp_key);
 578                         return sec_status_bogus;
 579                 }
 580                 dofree = 1;
 581         }
 582 #endif /* USE_ECDSA */
 583
 584         /* do the signature cryptography work */
 585 #ifdef HAVE_EVP_MD_CTX_NEW
 586         ctx = EVP_MD_CTX_new();
 587 #else
 588         ctx = (EVP_MD_CTX*)malloc(sizeof(*ctx));
 589         if(ctx) EVP_MD_CTX_init(ctx);
 590 #endif
 591         if(!ctx) {
 592                 log_err("EVP_MD_CTX_new: malloc failure");
 593                 EVP_PKEY_free(evp_key);
 594                 if(dofree) free(sigblock);
 595                 else if(docrypto_free) OPENSSL_free(sigblock);
 596                 return sec_status_unchecked;
 597         }
 598         if(EVP_VerifyInit(ctx, digest_type) == 0) {
 599                 verbose(VERB_QUERY, "verify: EVP_VerifyInit failed");
 600                 EVP_MD_CTX_destroy(ctx);
 601                 EVP_PKEY_free(evp_key);
 602                 if(dofree) free(sigblock);
 603                 else if(docrypto_free) OPENSSL_free(sigblock);
 604                 return sec_status_unchecked;
 605         }
 606         if(EVP_VerifyUpdate(ctx, (unsigned char*)sldns_buffer_begin(buf),
 607                 (unsigned int)sldns_buffer_limit(buf)) == 0) {
 608                 verbose(VERB_QUERY, "verify: EVP_VerifyUpdate failed");
 609                 EVP_MD_CTX_destroy(ctx);
 610                 EVP_PKEY_free(evp_key);
 611                 if(dofree) free(sigblock);
 612                 else if(docrypto_free) OPENSSL_free(sigblock);
 613                 return sec_status_unchecked;
 614         }
 615
 616         res = EVP_VerifyFinal(ctx, sigblock, sigblock_len, evp_key);
 617 #ifdef HAVE_EVP_MD_CTX_NEW
 618         EVP_MD_CTX_destroy(ctx);
 619 #else
 620         EVP_MD_CTX_cleanup(ctx);
 621         free(ctx);
 622 #endif
 623         EVP_PKEY_free(evp_key);
 624
 625         if(dofree) free(sigblock);
 626         else if(docrypto_free) OPENSSL_free(sigblock);
 627
 628         if(res == 1) {
 629                 return sec_status_secure;
 630         } else if(res == 0) {
 631                 verbose(VERB_QUERY, "verify: signature mismatch");
 632                 *reason = "signature crypto failed";
 633                 return sec_status_bogus;
 634         }
 635
 636         log_crypto_error("verify:", ERR_get_error());
 637         return sec_status_unchecked;
 638 }
 639
 640 /**************************************************/
 641 #elif defined(HAVE_NSS)
 642 /* libnss implementation */
 643 /* nss3 */
 644 #include "sechash.h"
 645 #include "pk11pub.h"
 646 #include "keyhi.h"
 647 #include "secerr.h"
 648 #include "cryptohi.h"
 649 /* nspr4 */
 650 #include "prerror.h"
 651
 652 /* return size of digest if supported, or 0 otherwise */
 653 size_t
 654 nsec3_hash_algo_size_supported(int id)
 655 {
 656         switch(id) {
 657         case NSEC3_HASH_SHA1:
 658                 return SHA1_LENGTH;
 659         default:
 660                 return 0;
 661         }
 662 }
 663
 664 /* perform nsec3 hash. return false on failure */
 665 int
 666 secalgo_nsec3_hash(int algo, unsigned char* buf, size_t len,
 667         unsigned char* res)
 668 {
 669         switch(algo) {
 670         case NSEC3_HASH_SHA1:
 671                 (void)HASH_HashBuf(HASH_AlgSHA1, res, buf, (unsigned long)len);
 672                 return 1;
 673         default:
 674                 return 0;
 675         }
 676 }
 677
 678 void
 679 secalgo_hash_sha256(unsigned char* buf, size_t len, unsigned char* res)
 680 {
 681         (void)HASH_HashBuf(HASH_AlgSHA256, res, buf, (unsigned long)len);
 682 }
 683
 684 size_t
 685 ds_digest_size_supported(int algo)
 686 {
 687         /* uses libNSS */
 688         switch(algo) {
 689                 case LDNS_SHA1:
 690                         return SHA1_LENGTH;
 691 #ifdef USE_SHA2
 692                 case LDNS_SHA256:
 693                         return SHA256_LENGTH;
 694 #endif
 695 #ifdef USE_ECDSA
 696                 case LDNS_SHA384:
 697                         return SHA384_LENGTH;
 698 #endif
 699                 /* GOST not supported in NSS */
 700                 case LDNS_HASH_GOST:
 701                 default: break;
 702         }
 703         return 0;
 704 }
 705
 706 int
 707 secalgo_ds_digest(int algo, unsigned char* buf, size_t len,
 708         unsigned char* res)
 709 {
 710         /* uses libNSS */
 711         switch(algo) {
 712                 case LDNS_SHA1:
 713                         return HASH_HashBuf(HASH_AlgSHA1, res, buf, len)
 714                                 == SECSuccess;
 715 #if defined(USE_SHA2)
 716                 case LDNS_SHA256:
 717                         return HASH_HashBuf(HASH_AlgSHA256, res, buf, len)
 718                                 == SECSuccess;
 719 #endif
 720 #ifdef USE_ECDSA
 721                 case LDNS_SHA384:
 722                         return HASH_HashBuf(HASH_AlgSHA384, res, buf, len)
 723                                 == SECSuccess;
 724 #endif
 725                 case LDNS_HASH_GOST:
 726                 default:
 727                         verbose(VERB_QUERY, "unknown DS digest algorithm %d",
 728                                 algo);
 729                         break;
 730         }
 731         return 0;
 732 }
 733
 734 int
 735 dnskey_algo_id_is_supported(int id)
 736 {
 737         /* uses libNSS */
 738         switch(id) {
 739         case LDNS_RSAMD5:
 740                 /* RFC 6725 deprecates RSAMD5 */
 741                 return 0;
 742 #ifdef USE_DSA
 743         case LDNS_DSA:
 744         case LDNS_DSA_NSEC3:
 745 #endif
 746         case LDNS_RSASHA1:
 747         case LDNS_RSASHA1_NSEC3:
 748 #ifdef USE_SHA2
 749         case LDNS_RSASHA256:
 750 #endif
 751 #ifdef USE_SHA2
 752         case LDNS_RSASHA512:
 753 #endif
 754                 return 1;
 755 #ifdef USE_ECDSA
 756         case LDNS_ECDSAP256SHA256:
 757         case LDNS_ECDSAP384SHA384:
 758                 return PK11_TokenExists(CKM_ECDSA);
 759 #endif
 760         case LDNS_ECC_GOST:
 761         default:
 762                 return 0;
 763         }
 764 }
 765
 766 /* return a new public key for NSS */
 767 static SECKEYPublicKey* nss_key_create(KeyType ktype)
 768 {
 769         SECKEYPublicKey* key;
 770         PLArenaPool* arena = PORT_NewArena(DER_DEFAULT_CHUNKSIZE);
 771         if(!arena) {
 772                 log_err("out of memory, PORT_NewArena failed");
 773                 return NULL;
 774         }
 775         key = PORT_ArenaZNew(arena, SECKEYPublicKey);
 776         if(!key) {
 777                 log_err("out of memory, PORT_ArenaZNew failed");
 778                 PORT_FreeArena(arena, PR_FALSE);
 779                 return NULL;
 780         }
 781         key->arena = arena;
 782         key->keyType = ktype;
 783         key->pkcs11Slot = NULL;
 784         key->pkcs11ID = CK_INVALID_HANDLE;
 785         return key;
 786 }
 787
 788 static SECKEYPublicKey* nss_buf2ecdsa(unsigned char* key, size_t len, int algo)
 789 {
 790         SECKEYPublicKey* pk;
 791         SECItem pub = {siBuffer, NULL, 0};
 792         SECItem params = {siBuffer, NULL, 0};
 793         static unsigned char param256[] = {
 794                 /* OBJECTIDENTIFIER 1.2.840.10045.3.1.7 (P-256)
 795                  * {iso(1) member-body(2) us(840) ansi-x962(10045) curves(3) prime(1) prime256v1(7)} */
 796                 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07
 797         };
 798         static unsigned char param384[] = {
 799                 /* OBJECTIDENTIFIER 1.3.132.0.34 (P-384)
 800                  * {iso(1) identified-organization(3) certicom(132) curve(0) ansip384r1(34)} */
 801                 0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x22
 802         };
 803         unsigned char buf[256+2]; /* sufficient for 2*384/8+1 */
 804
 805         /* check length, which uncompressed must be 2 bignums */
 806         if(algo == LDNS_ECDSAP256SHA256) {
 807                 if(len != 2*256/8) return NULL;
 808                 /* ECCurve_X9_62_PRIME_256V1 */
 809         } else if(algo == LDNS_ECDSAP384SHA384) {
 810                 if(len != 2*384/8) return NULL;
 811                 /* ECCurve_X9_62_PRIME_384R1 */
 812         } else    return NULL;
 813
 814         buf[0] = 0x04; /* POINT_FORM_UNCOMPRESSED */
 815         memmove(buf+1, key, len);
 816         pub.data = buf;
 817         pub.len = len+1;
 818         if(algo == LDNS_ECDSAP256SHA256) {
 819                 params.data = param256;
 820                 params.len = sizeof(param256);
 821         } else {
 822                 params.data = param384;
 823                 params.len = sizeof(param384);
 824         }
 825
 826         pk = nss_key_create(ecKey);
 827         if(!pk)
 828                 return NULL;
 829         pk->u.ec.size = (len/2)*8;
 830         if(SECITEM_CopyItem(pk->arena, &pk->u.ec.publicValue, &pub)) {
 831                 SECKEY_DestroyPublicKey(pk);
 832                 return NULL;
 833         }
 834         if(SECITEM_CopyItem(pk->arena, &pk->u.ec.DEREncodedParams, &params)) {
 835                 SECKEY_DestroyPublicKey(pk);
 836                 return NULL;
 837         }
 838
 839         return pk;
 840 }
 841
 842 static SECKEYPublicKey* nss_buf2dsa(unsigned char* key, size_t len)
 843 {
 844         SECKEYPublicKey* pk;
 845         uint8_t T;
 846         uint16_t length;
 847         uint16_t offset;
 848         SECItem Q = {siBuffer, NULL, 0};
 849         SECItem P = {siBuffer, NULL, 0};
 850         SECItem G = {siBuffer, NULL, 0};
 851         SECItem Y = {siBuffer, NULL, 0};
 852
 853         if(len == 0)
 854                 return NULL;
 855         T = (uint8_t)key[0];
 856         length = (64 + T * 8);
 857         offset = 1;
 858
 859         if (T > 8) {
 860                 return NULL;
 861         }
 862         if(len < (size_t)1 + SHA1_LENGTH + 3*length)
 863                 return NULL;
 864
 865         Q.data = key+offset;
 866         Q.len = SHA1_LENGTH;
 867         offset += SHA1_LENGTH;
 868
 869         P.data = key+offset;
 870         P.len = length;
 871         offset += length;
 872
 873         G.data = key+offset;
 874         G.len = length;
 875         offset += length;
 876
 877         Y.data = key+offset;
 878         Y.len = length;
 879         offset += length;
 880
 881         pk = nss_key_create(dsaKey);
 882         if(!pk)
 883                 return NULL;
 884         if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.params.prime, &P)) {
 885                 SECKEY_DestroyPublicKey(pk);
 886                 return NULL;
 887         }
 888         if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.params.subPrime, &Q)) {
 889                 SECKEY_DestroyPublicKey(pk);
 890                 return NULL;
 891         }
 892         if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.params.base, &G)) {
 893                 SECKEY_DestroyPublicKey(pk);
 894                 return NULL;
 895         }
 896         if(SECITEM_CopyItem(pk->arena, &pk->u.dsa.publicValue, &Y)) {
 897                 SECKEY_DestroyPublicKey(pk);
 898                 return NULL;
 899         }
 900         return pk;
 901 }
 902
 903 static SECKEYPublicKey* nss_buf2rsa(unsigned char* key, size_t len)
 904 {
 905         SECKEYPublicKey* pk;
 906         uint16_t exp;
 907         uint16_t offset;
 908         uint16_t int16;
 909         SECItem modulus = {siBuffer, NULL, 0};
 910         SECItem exponent = {siBuffer, NULL, 0};
 911         if(len == 0)
 912                 return NULL;
 913         if(key[0] == 0) {
 914                 if(len < 3)
 915                         return NULL;
 916                 /* the exponent is too large so it's places further */
 917                 memmove(&int16, key+1, 2);
 918                 exp = ntohs(int16);
 919                 offset = 3;
 920         } else {
 921                 exp = key[0];
 922                 offset = 1;
 923         }
 924
 925         /* key length at least one */
 926         if(len < (size_t)offset + exp + 1)
 927                 return NULL;
 928
 929         exponent.data = key+offset;
 930         exponent.len = exp;
 931         offset += exp;
 932         modulus.data = key+offset;
 933         modulus.len = (len - offset);
 934
 935         pk = nss_key_create(rsaKey);
 936         if(!pk)
 937                 return NULL;
 938         if(SECITEM_CopyItem(pk->arena, &pk->u.rsa.modulus, &modulus)) {
 939                 SECKEY_DestroyPublicKey(pk);
 940                 return NULL;
 941         }
 942         if(SECITEM_CopyItem(pk->arena, &pk->u.rsa.publicExponent, &exponent)) {
 943                 SECKEY_DestroyPublicKey(pk);
 944                 return NULL;
 945         }
 946         return pk;
 947 }
 948
 949 /**
 950  * Setup key and digest for verification. Adjust sig if necessary.
 951  *
 952  * @param algo: key algorithm
 953  * @param evp_key: EVP PKEY public key to create.
 954  * @param digest_type: digest type to use
 955  * @param key: key to setup for.
 956  * @param keylen: length of key.
 957  * @param prefix: if returned, the ASN prefix for the hashblob.
 958  * @param prefixlen: length of the prefix.
 959  * @return false on failure.
 960  */
 961 static int
 962 nss_setup_key_digest(int algo, SECKEYPublicKey** pubkey, HASH_HashType* htype,
 963         unsigned char* key, size_t keylen, unsigned char** prefix,
 964         size_t* prefixlen)
 965 {
 966         /* uses libNSS */
 967
 968         /* hash prefix for md5, RFC2537 */
 969         static unsigned char p_md5[] = {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a,
 970         0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10};
 971         /* hash prefix to prepend to hash output, from RFC3110 */
 972         static unsigned char p_sha1[] = {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2B,
 973                 0x0E, 0x03, 0x02, 0x1A, 0x05, 0x00, 0x04, 0x14};
 974         /* from RFC5702 */
 975         static unsigned char p_sha256[] = {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60,
 976         0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
 977         static unsigned char p_sha512[] = {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60,
 978         0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
 979         /* from RFC6234 */
 980         /* for future RSASHA384 ..
 981         static unsigned char p_sha384[] = {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60,
 982         0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30};
 983         */
 984
 985         switch(algo) {
 986 #ifdef USE_DSA
 987                 case LDNS_DSA:
 988                 case LDNS_DSA_NSEC3:
 989                         *pubkey = nss_buf2dsa(key, keylen);
 990                         if(!*pubkey) {
 991                                 log_err("verify: malloc failure in crypto");
 992                                 return 0;
 993                         }
 994                         *htype = HASH_AlgSHA1;
 995                         /* no prefix for DSA verification */
 996                         break;
 997 #endif
 998                 case LDNS_RSASHA1:
 999                 case LDNS_RSASHA1_NSEC3:
1000 #ifdef USE_SHA2
1001                 case LDNS_RSASHA256:
1002 #endif
1003 #ifdef USE_SHA2
1004                 case LDNS_RSASHA512:
1005 #endif
1006                         *pubkey = nss_buf2rsa(key, keylen);
1007                         if(!*pubkey) {
1008                                 log_err("verify: malloc failure in crypto");
1009                                 return 0;
1010                         }
1011                         /* select SHA version */
1012 #ifdef USE_SHA2
1013                         if(algo == LDNS_RSASHA256) {
1014                                 *htype = HASH_AlgSHA256;
1015                                 *prefix = p_sha256;
1016                                 *prefixlen = sizeof(p_sha256);
1017                         } else
1018 #endif
1019 #ifdef USE_SHA2
1020                                 if(algo == LDNS_RSASHA512) {
1021                                 *htype = HASH_AlgSHA512;
1022                                 *prefix = p_sha512;
1023                                 *prefixlen = sizeof(p_sha512);
1024                         } else
1025 #endif
1026                         {
1027                                 *htype = HASH_AlgSHA1;
1028                                 *prefix = p_sha1;
1029                                 *prefixlen = sizeof(p_sha1);
1030                         }
1031
1032                         break;
1033                 case LDNS_RSAMD5:
1034                         *pubkey = nss_buf2rsa(key, keylen);
1035                         if(!*pubkey) {
1036                                 log_err("verify: malloc failure in crypto");
1037                                 return 0;
1038                         }
1039                         *htype = HASH_AlgMD5;
1040                         *prefix = p_md5;
1041                         *prefixlen = sizeof(p_md5);
1042
1043                         break;
1044 #ifdef USE_ECDSA
1045                 case LDNS_ECDSAP256SHA256:
1046                         *pubkey = nss_buf2ecdsa(key, keylen,
1047                                 LDNS_ECDSAP256SHA256);
1048                         if(!*pubkey) {
1049                                 log_err("verify: malloc failure in crypto");
1050                                 return 0;
1051                         }
1052                         *htype = HASH_AlgSHA256;
1053                         /* no prefix for DSA verification */
1054                         break;
1055                 case LDNS_ECDSAP384SHA384:
1056                         *pubkey = nss_buf2ecdsa(key, keylen,
1057                                 LDNS_ECDSAP384SHA384);
1058                         if(!*pubkey) {
1059                                 log_err("verify: malloc failure in crypto");
1060                                 return 0;
1061                         }
1062                         *htype = HASH_AlgSHA384;
1063                         /* no prefix for DSA verification */
1064                         break;
1065 #endif /* USE_ECDSA */
1066                 case LDNS_ECC_GOST:
1067                 default:
1068                         verbose(VERB_QUERY, "verify: unknown algorithm %d",
1069                                 algo);
1070                         return 0;
1071         }
1072         return 1;
1073 }
1074
1075 /**
1076  * Check a canonical sig+rrset and signature against a dnskey
1077  * @param buf: buffer with data to verify, the first rrsig part and the
1078  *      canonicalized rrset.
1079  * @param algo: DNSKEY algorithm.
1080  * @param sigblock: signature rdata field from RRSIG
1081  * @param sigblock_len: length of sigblock data.
1082  * @param key: public key data from DNSKEY RR.
1083  * @param keylen: length of keydata.
1084  * @param reason: bogus reason in more detail.
1085  * @return secure if verification succeeded, bogus on crypto failure,
1086  *      unchecked on format errors and alloc failures.
1087  */
1088 enum sec_status
1089 verify_canonrrset(sldns_buffer* buf, int algo, unsigned char* sigblock,
1090         unsigned int sigblock_len, unsigned char* key, unsigned int keylen,
1091         char** reason)
1092 {
1093         /* uses libNSS */
1094         /* large enough for the different hashes */
1095         unsigned char hash[HASH_LENGTH_MAX];
1096         unsigned char hash2[HASH_LENGTH_MAX*2];
1097         HASH_HashType htype = 0;
1098         SECKEYPublicKey* pubkey = NULL;
1099         SECItem secsig = {siBuffer, sigblock, sigblock_len};
1100         SECItem sechash = {siBuffer, hash, 0};
1101         SECStatus res;
1102         unsigned char* prefix = NULL; /* prefix for hash, RFC3110, RFC5702 */
1103         size_t prefixlen = 0;
1104         int err;
1105
1106         if(!nss_setup_key_digest(algo, &pubkey, &htype, key, keylen,
1107                 &prefix, &prefixlen)) {
1108                 verbose(VERB_QUERY, "verify: failed to setup key");
1109                 *reason = "use of key for crypto failed";
1110                 SECKEY_DestroyPublicKey(pubkey);
1111                 return sec_status_bogus;
1112         }
1113
1114 #ifdef USE_DSA
1115         /* need to convert DSA, ECDSA signatures? */
1116         if((algo == LDNS_DSA || algo == LDNS_DSA_NSEC3)) {
1117                 if(sigblock_len == 1+2*SHA1_LENGTH) {
1118                         secsig.data ++;
1119                         secsig.len --;
1120                 } else {
1121                         SECItem* p = DSAU_DecodeDerSig(&secsig);
1122                         if(!p) {
1123                                 verbose(VERB_QUERY, "verify: failed DER decode");
1124                                 *reason = "signature DER decode failed";
1125                                 SECKEY_DestroyPublicKey(pubkey);
1126                                 return sec_status_bogus;
1127                         }
1128                         if(SECITEM_CopyItem(pubkey->arena, &secsig, p)) {
1129                                 log_err("alloc failure in DER decode");
1130                                 SECKEY_DestroyPublicKey(pubkey);
1131                                 SECITEM_FreeItem(p, PR_TRUE);
1132                                 return sec_status_unchecked;
1133                         }
1134                         SECITEM_FreeItem(p, PR_TRUE);
1135                 }
1136         }
1137 #endif /* USE_DSA */
1138
1139         /* do the signature cryptography work */
1140         /* hash the data */
1141         sechash.len = HASH_ResultLen(htype);
1142         if(sechash.len > sizeof(hash)) {
1143                 verbose(VERB_QUERY, "verify: hash too large for buffer");
1144                 SECKEY_DestroyPublicKey(pubkey);
1145                 return sec_status_unchecked;
1146         }
1147         if(HASH_HashBuf(htype, hash, (unsigned char*)sldns_buffer_begin(buf),
1148                 (unsigned int)sldns_buffer_limit(buf)) != SECSuccess) {
1149                 verbose(VERB_QUERY, "verify: HASH_HashBuf failed");
1150                 SECKEY_DestroyPublicKey(pubkey);
1151                 return sec_status_unchecked;
1152         }
1153         if(prefix) {
1154                 int hashlen = sechash.len;
1155                 if(prefixlen+hashlen > sizeof(hash2)) {
1156                         verbose(VERB_QUERY, "verify: hashprefix too large");
1157                         SECKEY_DestroyPublicKey(pubkey);
1158                         return sec_status_unchecked;
1159                 }
1160                 sechash.data = hash2;
1161                 sechash.len = prefixlen+hashlen;
1162                 memcpy(sechash.data, prefix, prefixlen);
1163                 memmove(sechash.data+prefixlen, hash, hashlen);
1164         }
1165
1166         /* verify the signature */
1167         res = PK11_Verify(pubkey, &secsig, &sechash, NULL /*wincx*/);
1168         SECKEY_DestroyPublicKey(pubkey);
1169
1170         if(res == SECSuccess) {
1171                 return sec_status_secure;
1172         }
1173         err = PORT_GetError();
1174         if(err != SEC_ERROR_BAD_SIGNATURE) {
1175                 /* failed to verify */
1176                 verbose(VERB_QUERY, "verify: PK11_Verify failed: %s",
1177                         PORT_ErrorToString(err));
1178                 /* if it is not supported, like ECC is removed, we get,
1179                  * SEC_ERROR_NO_MODULE */
1180                 if(err == SEC_ERROR_NO_MODULE)
1181                         return sec_status_unchecked;
1182                 /* but other errors are commonly returned
1183                  * for a bad signature from NSS.  Thus we return bogus,
1184                  * not unchecked */
1185                 *reason = "signature crypto failed";
1186                 return sec_status_bogus;
1187         }
1188         verbose(VERB_QUERY, "verify: signature mismatch: %s",
1189                 PORT_ErrorToString(err));
1190         *reason = "signature crypto failed";
1191         return sec_status_bogus;
1192 }
1193
1194 #elif defined(HAVE_NETTLE)
1195
1196 #include "sha.h"
1197 #include "bignum.h"
1198 #include "macros.h"
1199 #include "rsa.h"
1200 #include "dsa.h"
1201 #ifdef HAVE_NETTLE_DSA_COMPAT_H
1202 #include "dsa-compat.h"
1203 #endif
1204 #include "asn1.h"
1205 #ifdef USE_ECDSA
1206 #include "ecdsa.h"
1207 #include "ecc-curve.h"
1208 #endif
1209
1210 static int
1211 _digest_nettle(int algo, uint8_t* buf, size_t len,
1212         unsigned char* res)
1213 {
1214         switch(algo) {
1215                 case SHA1_DIGEST_SIZE:
1216                 {
1217                         struct sha1_ctx ctx;
1218                         sha1_init(&ctx);
1219                         sha1_update(&ctx, len, buf);
1220                         sha1_digest(&ctx, SHA1_DIGEST_SIZE, res);
1221                         return 1;
1222                 }
1223                 case SHA256_DIGEST_SIZE:
1224                 {
1225                         struct sha256_ctx ctx;
1226                         sha256_init(&ctx);
1227                         sha256_update(&ctx, len, buf);
1228                         sha256_digest(&ctx, SHA256_DIGEST_SIZE, res);
1229                         return 1;
1230                 }
1231                 case SHA384_DIGEST_SIZE:
1232                 {
1233                         struct sha384_ctx ctx;
1234                         sha384_init(&ctx);
1235                         sha384_update(&ctx, len, buf);
1236                         sha384_digest(&ctx, SHA384_DIGEST_SIZE, res);
1237                         return 1;
1238                 }
1239                 case SHA512_DIGEST_SIZE:
1240                 {
1241                         struct sha512_ctx ctx;
1242                         sha512_init(&ctx);
1243                         sha512_update(&ctx, len, buf);
1244                         sha512_digest(&ctx, SHA512_DIGEST_SIZE, res);
1245                         return 1;
1246                 }
1247                 default:
1248                         break;
1249         }
1250         return 0;
1251 }
1252
1253 /* return size of digest if supported, or 0 otherwise */
1254 size_t
1255 nsec3_hash_algo_size_supported(int id)
1256 {
1257         switch(id) {
1258         case NSEC3_HASH_SHA1:
1259                 return SHA1_DIGEST_SIZE;
1260         default:
1261                 return 0;
1262         }
1263 }
1264
1265 /* perform nsec3 hash. return false on failure */
1266 int
1267 secalgo_nsec3_hash(int algo, unsigned char* buf, size_t len,
1268         unsigned char* res)
1269 {
1270         switch(algo) {
1271         case NSEC3_HASH_SHA1:
1272                 return _digest_nettle(SHA1_DIGEST_SIZE, (uint8_t*)buf, len,
1273                         res);
1274         default:
1275                 return 0;
1276         }
1277 }
1278
1279 void
1280 secalgo_hash_sha256(unsigned char* buf, size_t len, unsigned char* res)
1281 {
1282         _digest_nettle(SHA256_DIGEST_SIZE, (uint8_t*)buf, len, res);
1283 }
1284
1285 /**
1286  * Return size of DS digest according to its hash algorithm.
1287  * @param algo: DS digest algo.
1288  * @return size in bytes of digest, or 0 if not supported.
1289  */
1290 size_t
1291 ds_digest_size_supported(int algo)
1292 {
1293         switch(algo) {
1294                 case LDNS_SHA1:
1295                         return SHA1_DIGEST_SIZE;
1296 #ifdef USE_SHA2
1297                 case LDNS_SHA256:
1298                         return SHA256_DIGEST_SIZE;
1299 #endif
1300 #ifdef USE_ECDSA
1301                 case LDNS_SHA384:
1302                         return SHA384_DIGEST_SIZE;
1303 #endif
1304                 /* GOST not supported */
1305                 case LDNS_HASH_GOST:
1306                 default:
1307                         break;
1308         }
1309         return 0;
1310 }
1311
1312 int
1313 secalgo_ds_digest(int algo, unsigned char* buf, size_t len,
1314         unsigned char* res)
1315 {
1316         switch(algo) {
1317                 case LDNS_SHA1:
1318                         return _digest_nettle(SHA1_DIGEST_SIZE, buf, len, res);
1319 #if defined(USE_SHA2)
1320                 case LDNS_SHA256:
1321                         return _digest_nettle(SHA256_DIGEST_SIZE, buf, len, res);
1322 #endif
1323 #ifdef USE_ECDSA
1324                 case LDNS_SHA384:
1325                         return _digest_nettle(SHA384_DIGEST_SIZE, buf, len, res);
1326
1327 #endif
1328                 case LDNS_HASH_GOST:
1329                 default:
1330                         verbose(VERB_QUERY, "unknown DS digest algorithm %d",
1331                                 algo);
1332                         break;
1333         }
1334         return 0;
1335 }
1336
1337 int
1338 dnskey_algo_id_is_supported(int id)
1339 {
1340         /* uses libnettle */
1341         switch(id) {
1342 #ifdef USE_DSA
1343         case LDNS_DSA:
1344         case LDNS_DSA_NSEC3:
1345 #endif
1346         case LDNS_RSASHA1:
1347         case LDNS_RSASHA1_NSEC3:
1348 #ifdef USE_SHA2
1349         case LDNS_RSASHA256:
1350         case LDNS_RSASHA512:
1351 #endif
1352 #ifdef USE_ECDSA
1353         case LDNS_ECDSAP256SHA256:
1354         case LDNS_ECDSAP384SHA384:
1355 #endif
1356                 return 1;
1357         case LDNS_RSAMD5: /* RFC 6725 deprecates RSAMD5 */
1358         case LDNS_ECC_GOST:
1359         default:
1360                 return 0;
1361         }
1362 }
1363
1364 #ifdef USE_DSA
1365 static char *
1366 _verify_nettle_dsa(sldns_buffer* buf, unsigned char* sigblock,
1367         unsigned int sigblock_len, unsigned char* key, unsigned int keylen)
1368 {
1369         uint8_t digest[SHA1_DIGEST_SIZE];
1370         uint8_t key_t;
1371         int res = 0;
1372         size_t offset;
1373         struct dsa_public_key pubkey;
1374         struct dsa_signature signature;
1375         unsigned int expected_len;
1376
1377         /* Extract DSA signature from the record */
1378         nettle_dsa_signature_init(&signature);
1379         /* Signature length: 41 bytes - RFC 2536 sec. 3 */
1380         if(sigblock_len == 41) {
1381                 if(key[0] != sigblock[0])
1382                         return "invalid T value in DSA signature or pubkey";
1383                 nettle_mpz_set_str_256_u(signature.r, 20, sigblock+1);
1384                 nettle_mpz_set_str_256_u(signature.s, 20, sigblock+1+20);
1385         } else {
1386                 /* DER encoded, decode the ASN1 notated R and S bignums */
1387                 /* SEQUENCE { r INTEGER, s INTEGER } */
1388                 struct asn1_der_iterator i, seq;
1389                 if(asn1_der_iterator_first(&i, sigblock_len,
1390                         (uint8_t*)sigblock) != ASN1_ITERATOR_CONSTRUCTED
1391                         || i.type != ASN1_SEQUENCE)
1392                         return "malformed DER encoded DSA signature";
1393                 /* decode this element of i using the seq iterator */
1394                 if(asn1_der_decode_constructed(&i, &seq) !=
1395                         ASN1_ITERATOR_PRIMITIVE || seq.type != ASN1_INTEGER)
1396                         return "malformed DER encoded DSA signature";
1397                 if(!asn1_der_get_bignum(&seq, signature.r, 20*8))
1398                         return "malformed DER encoded DSA signature";
1399                 if(asn1_der_iterator_next(&seq) != ASN1_ITERATOR_PRIMITIVE
1400                         || seq.type != ASN1_INTEGER)
1401                         return "malformed DER encoded DSA signature";
1402                 if(!asn1_der_get_bignum(&seq, signature.s, 20*8))
1403                         return "malformed DER encoded DSA signature";
1404                 if(asn1_der_iterator_next(&i) != ASN1_ITERATOR_END)
1405                         return "malformed DER encoded DSA signature";
1406         }
1407
1408         /* Validate T values constraints - RFC 2536 sec. 2 & sec. 3 */
1409         key_t = key[0];
1410         if (key_t > 8) {
1411                 return "invalid T value in DSA pubkey";
1412         }
1413
1414         /* Pubkey minimum length: 21 bytes - RFC 2536 sec. 2 */
1415         if (keylen < 21) {
1416                 return "DSA pubkey too short";
1417         }
1418
1419         expected_len =   1 +            /* T */
1420                         20 +            /* Q */
1421                        (64 + key_t*8) + /* P */
1422                        (64 + key_t*8) + /* G */
1423                        (64 + key_t*8);  /* Y */
1424         if (keylen != expected_len ) {
1425                 return "invalid DSA pubkey length";
1426         }
1427
1428         /* Extract DSA pubkey from the record */
1429         nettle_dsa_public_key_init(&pubkey);
1430         offset = 1;
1431         nettle_mpz_set_str_256_u(pubkey.q, 20, key+offset);
1432         offset += 20;
1433         nettle_mpz_set_str_256_u(pubkey.p, (64 + key_t*8), key+offset);
1434         offset += (64 + key_t*8);
1435         nettle_mpz_set_str_256_u(pubkey.g, (64 + key_t*8), key+offset);
1436         offset += (64 + key_t*8);
1437         nettle_mpz_set_str_256_u(pubkey.y, (64 + key_t*8), key+offset);
1438
1439         /* Digest content of "buf" and verify its DSA signature in "sigblock"*/
1440         res = _digest_nettle(SHA1_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1441                                                 (unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1442         res &= dsa_sha1_verify_digest(&pubkey, digest, &signature);
1443
1444         /* Clear and return */
1445         nettle_dsa_signature_clear(&signature);
1446         nettle_dsa_public_key_clear(&pubkey);
1447         if (!res)
1448                 return "DSA signature verification failed";
1449         else
1450                 return NULL;
1451 }
1452 #endif /* USE_DSA */
1453
1454 static char *
1455 _verify_nettle_rsa(sldns_buffer* buf, unsigned int digest_size, char* sigblock,
1456         unsigned int sigblock_len, uint8_t* key, unsigned int keylen)
1457 {
1458         uint16_t exp_len = 0;
1459         size_t exp_offset = 0, mod_offset = 0;
1460         struct rsa_public_key pubkey;
1461         mpz_t signature;
1462         int res = 0;
1463
1464         /* RSA pubkey parsing as per RFC 3110 sec. 2 */
1465         if( keylen <= 1) {
1466                 return "null RSA key";
1467         }
1468         if (key[0] != 0) {
1469                 /* 1-byte length */
1470                 exp_len = key[0];
1471                 exp_offset = 1;
1472         } else {
1473                 /* 1-byte NUL + 2-bytes exponent length */
1474                 if (keylen < 3) {
1475                         return "incorrect RSA key length";
1476                 }
1477                 exp_len = READ_UINT16(key+1);
1478                 if (exp_len == 0)
1479                         return "null RSA exponent length";
1480                 exp_offset = 3;
1481         }
1482         /* Check that we are not over-running input length */
1483         if (keylen < exp_offset + exp_len + 1) {
1484                 return "RSA key content shorter than expected";
1485         }
1486         mod_offset = exp_offset + exp_len;
1487         nettle_rsa_public_key_init(&pubkey);
1488         pubkey.size = keylen - mod_offset;
1489         nettle_mpz_set_str_256_u(pubkey.e, exp_len, &key[exp_offset]);
1490         nettle_mpz_set_str_256_u(pubkey.n, pubkey.size, &key[mod_offset]);
1491
1492         /* Digest content of "buf" and verify its RSA signature in "sigblock"*/
1493         nettle_mpz_init_set_str_256_u(signature, sigblock_len, (uint8_t*)sigblock);
1494         switch (digest_size) {
1495                 case SHA1_DIGEST_SIZE:
1496                 {
1497                         uint8_t digest[SHA1_DIGEST_SIZE];
1498                         res = _digest_nettle(SHA1_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1499                                                 (unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1500                         res &= rsa_sha1_verify_digest(&pubkey, digest, signature);
1501                         break;
1502                 }
1503                 case SHA256_DIGEST_SIZE:
1504                 {
1505                         uint8_t digest[SHA256_DIGEST_SIZE];
1506                         res = _digest_nettle(SHA256_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1507                                                 (unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1508                         res &= rsa_sha256_verify_digest(&pubkey, digest, signature);
1509                         break;
1510                 }
1511                 case SHA512_DIGEST_SIZE:
1512                 {
1513                         uint8_t digest[SHA512_DIGEST_SIZE];
1514                         res = _digest_nettle(SHA512_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1515                                                 (unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1516                         res &= rsa_sha512_verify_digest(&pubkey, digest, signature);
1517                         break;
1518                 }
1519                 default:
1520                         break;
1521         }
1522
1523         /* Clear and return */
1524         nettle_rsa_public_key_clear(&pubkey);
1525         mpz_clear(signature);
1526         if (!res) {
1527                 return "RSA signature verification failed";
1528         } else {
1529                 return NULL;
1530         }
1531 }
1532
1533 #ifdef USE_ECDSA
1534 static char *
1535 _verify_nettle_ecdsa(sldns_buffer* buf, unsigned int digest_size, unsigned char* sigblock,
1536         unsigned int sigblock_len, unsigned char* key, unsigned int keylen)
1537 {
1538         int res = 0;
1539         struct ecc_point pubkey;
1540         struct dsa_signature signature;
1541
1542         /* Always matched strength, as per RFC 6605 sec. 1 */
1543         if (sigblock_len != 2*digest_size || keylen != 2*digest_size) {
1544                 return "wrong ECDSA signature length";
1545         }
1546
1547         /* Parse ECDSA signature as per RFC 6605 sec. 4 */
1548         nettle_dsa_signature_init(&signature);
1549         switch (digest_size) {
1550                 case SHA256_DIGEST_SIZE:
1551                 {
1552                         uint8_t digest[SHA256_DIGEST_SIZE];
1553                         mpz_t x, y;
1554                         nettle_ecc_point_init(&pubkey, &nettle_secp_256r1);
1555                         nettle_mpz_init_set_str_256_u(x, SHA256_DIGEST_SIZE, key);
1556                         nettle_mpz_init_set_str_256_u(y, SHA256_DIGEST_SIZE, key+SHA256_DIGEST_SIZE);
1557                         nettle_mpz_set_str_256_u(signature.r, SHA256_DIGEST_SIZE, sigblock);
1558                         nettle_mpz_set_str_256_u(signature.s, SHA256_DIGEST_SIZE, sigblock+SHA256_DIGEST_SIZE);
1559                         res = _digest_nettle(SHA256_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1560                                                 (unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1561                         res &= nettle_ecc_point_set(&pubkey, x, y);
1562                         res &= nettle_ecdsa_verify (&pubkey, SHA256_DIGEST_SIZE, digest, &signature);
1563                         mpz_clear(x);
1564                         mpz_clear(y);
1565                         break;
1566                 }
1567                 case SHA384_DIGEST_SIZE:
1568                 {
1569                         uint8_t digest[SHA384_DIGEST_SIZE];
1570                         mpz_t x, y;
1571                         nettle_ecc_point_init(&pubkey, &nettle_secp_384r1);
1572                         nettle_mpz_init_set_str_256_u(x, SHA384_DIGEST_SIZE, key);
1573                         nettle_mpz_init_set_str_256_u(y, SHA384_DIGEST_SIZE, key+SHA384_DIGEST_SIZE);
1574                         nettle_mpz_set_str_256_u(signature.r, SHA384_DIGEST_SIZE, sigblock);
1575                         nettle_mpz_set_str_256_u(signature.s, SHA384_DIGEST_SIZE, sigblock+SHA384_DIGEST_SIZE);
1576                         res = _digest_nettle(SHA384_DIGEST_SIZE, (unsigned char*)sldns_buffer_begin(buf),
1577                                                 (unsigned int)sldns_buffer_limit(buf), (unsigned char*)digest);
1578                         res &= nettle_ecc_point_set(&pubkey, x, y);
1579                         res &= nettle_ecdsa_verify (&pubkey, SHA384_DIGEST_SIZE, digest, &signature);
1580                         mpz_clear(x);
1581                         mpz_clear(y);
1582                         nettle_ecc_point_clear(&pubkey);
1583                         break;
1584                 }
1585                 default:
1586                         return "unknown ECDSA algorithm";
1587         }
1588
1589         /* Clear and return */
1590         nettle_dsa_signature_clear(&signature);
1591         if (!res)
1592                 return "ECDSA signature verification failed";
1593         else
1594                 return NULL;
1595 }
1596 #endif
1597
1598 /**
1599  * Check a canonical sig+rrset and signature against a dnskey
1600  * @param buf: buffer with data to verify, the first rrsig part and the
1601  *      canonicalized rrset.
1602  * @param algo: DNSKEY algorithm.
1603  * @param sigblock: signature rdata field from RRSIG
1604  * @param sigblock_len: length of sigblock data.
1605  * @param key: public key data from DNSKEY RR.
1606  * @param keylen: length of keydata.
1607  * @param reason: bogus reason in more detail.
1608  * @return secure if verification succeeded, bogus on crypto failure,
1609  *      unchecked on format errors and alloc failures.
1610  */
1611 enum sec_status
1612 verify_canonrrset(sldns_buffer* buf, int algo, unsigned char* sigblock,
1613         unsigned int sigblock_len, unsigned char* key, unsigned int keylen,
1614         char** reason)
1615 {
1616         unsigned int digest_size = 0;
1617
1618         if (sigblock_len == 0 || keylen == 0) {
1619                 *reason = "null signature";
1620                 return sec_status_bogus;
1621         }
1622
1623         switch(algo) {
1624 #ifdef USE_DSA
1625         case LDNS_DSA:
1626         case LDNS_DSA_NSEC3:
1627                 *reason = _verify_nettle_dsa(buf, sigblock, sigblock_len, key, keylen);
1628                 if (*reason != NULL)
1629                         return sec_status_bogus;
1630                 else
1631                         return sec_status_secure;
1632 #endif /* USE_DSA */
1633
1634         case LDNS_RSASHA1:
1635         case LDNS_RSASHA1_NSEC3:
1636                 digest_size = (digest_size ? digest_size : SHA1_DIGEST_SIZE);
1637 #ifdef USE_SHA2
1638         case LDNS_RSASHA256:
1639                 digest_size = (digest_size ? digest_size : SHA256_DIGEST_SIZE);
1640         case LDNS_RSASHA512:
1641                 digest_size = (digest_size ? digest_size : SHA512_DIGEST_SIZE);
1642
1643 #endif
1644                 *reason = _verify_nettle_rsa(buf, digest_size, (char*)sigblock,
1645                                                 sigblock_len, key, keylen);
1646                 if (*reason != NULL)
1647                         return sec_status_bogus;
1648                 else
1649                         return sec_status_secure;
1650
1651 #ifdef USE_ECDSA
1652         case LDNS_ECDSAP256SHA256:
1653                 digest_size = (digest_size ? digest_size : SHA256_DIGEST_SIZE);
1654         case LDNS_ECDSAP384SHA384:
1655                 digest_size = (digest_size ? digest_size : SHA384_DIGEST_SIZE);
1656                 *reason = _verify_nettle_ecdsa(buf, digest_size, sigblock,
1657                                                 sigblock_len, key, keylen);
1658                 if (*reason != NULL)
1659                         return sec_status_bogus;
1660                 else
1661                         return sec_status_secure;
1662 #endif
1663         case LDNS_RSAMD5:
1664         case LDNS_ECC_GOST:
1665         default:
1666                 *reason = "unable to verify signature, unknown algorithm";
1667                 return sec_status_bogus;
1668         }
1669 }
1670
1671 #endif /* HAVE_SSL or HAVE_NSS or HAVE_NETTLE */