contrib/bind9/lib/dns/opensslrsa_link.c

   1 /*
   2  * Copyright (C) 2004-2009, 2011-2014  Internet Systems Consortium, Inc. ("ISC")
   3  * Copyright (C) 2000-2003  Internet Software Consortium.
   4  *
   5  * Permission to use, copy, modify, and/or distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
  10  * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
  11  * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
  12  * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  13  * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
  14  * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  15  * PERFORMANCE OF THIS SOFTWARE.
  16  */
  17
  18 /*
  19  * Principal Author: Brian Wellington
  20  * $Id$
  21  */
  22 #ifdef OPENSSL
  23 #include <config.h>
  24
  25 #ifndef USE_EVP
  26 #if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
  27 #define USE_EVP 0
  28 #else
  29 #define USE_EVP 1
  30 #endif
  31 #endif
  32
  33
  34 #include <isc/entropy.h>
  35 #include <isc/md5.h>
  36 #include <isc/sha1.h>
  37 #include <isc/sha2.h>
  38 #include <isc/mem.h>
  39 #include <isc/string.h>
  40 #include <isc/util.h>
  41
  42 #include <dst/result.h>
  43
  44 #include "dst_internal.h"
  45 #include "dst_openssl.h"
  46 #include "dst_parse.h"
  47
  48 #include <openssl/err.h>
  49 #include <openssl/objects.h>
  50 #include <openssl/rsa.h>
  51 #if OPENSSL_VERSION_NUMBER > 0x00908000L
  52 #include <openssl/bn.h>
  53 #endif
  54 #ifdef USE_ENGINE
  55 #include <openssl/engine.h>
  56 #endif
  57
  58 /*
  59  * Limit the size of public exponents.
  60  */
  61 #ifndef RSA_MAX_PUBEXP_BITS
  62 #define RSA_MAX_PUBEXP_BITS    35
  63 #endif
  64
  65 /*
  66  * We don't use configure for windows so enforce the OpenSSL version
  67  * here.  Unlike with configure we don't support overriding this test.
  68  */
  69 #ifdef WIN32
  70 #if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
  71        OPENSSL_VERSION_NUMBER < 0x00908000L) || \
  72       OPENSSL_VERSION_NUMBER >= 0x0090804fL)
  73 #error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
  74 #endif
  75 #endif
  76
  77
  78         /*
  79          * XXXMPA  Temporarily disable RSA_BLINDING as it requires
  80          * good quality random data that cannot currently be guaranteed.
  81          * XXXMPA  Find which versions of openssl use pseudo random data
  82          * and set RSA_FLAG_BLINDING for those.
  83          */
  84
  85 #if 0
  86 #if OPENSSL_VERSION_NUMBER < 0x0090601fL
  87 #define SET_FLAGS(rsa) \
  88         do { \
  89         (rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
  90         (rsa)->flags |= RSA_FLAG_BLINDING; \
  91         } while (0)
  92 #else
  93 #define SET_FLAGS(rsa) \
  94         do { \
  95                 (rsa)->flags |= RSA_FLAG_BLINDING; \
  96         } while (0)
  97 #endif
  98 #endif
  99
 100 #if OPENSSL_VERSION_NUMBER < 0x0090601fL
 101 #define SET_FLAGS(rsa) \
 102         do { \
 103         (rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
 104         (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 105         } while (0)
 106 #elif defined(RSA_FLAG_NO_BLINDING)
 107 #define SET_FLAGS(rsa) \
 108         do { \
 109                 (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 110                 (rsa)->flags |= RSA_FLAG_NO_BLINDING; \
 111         } while (0)
 112 #else
 113 #define SET_FLAGS(rsa) \
 114         do { \
 115                 (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 116         } while (0)
 117 #endif
 118
 119 #define DST_RET(a) {ret = a; goto err;}
 120
 121 static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);
 122
 123 static isc_result_t
 124 opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
 125 #if USE_EVP
 126         EVP_MD_CTX *evp_md_ctx;
 127         const EVP_MD *type = NULL;
 128 #endif
 129
 130         UNUSED(key);
 131         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 132                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 133                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 134                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 135                 dctx->key->key_alg == DST_ALG_RSASHA512);
 136
 137 #if USE_EVP
 138         evp_md_ctx = EVP_MD_CTX_create();
 139         if (evp_md_ctx == NULL)
 140                 return (ISC_R_NOMEMORY);
 141
 142         switch (dctx->key->key_alg) {
 143         case DST_ALG_RSAMD5:
 144                 type = EVP_md5();       /* MD5 + RSA */
 145                 break;
 146         case DST_ALG_RSASHA1:
 147         case DST_ALG_NSEC3RSASHA1:
 148                 type = EVP_sha1();      /* SHA1 + RSA */
 149                 break;
 150 #ifdef HAVE_EVP_SHA256
 151         case DST_ALG_RSASHA256:
 152                 type = EVP_sha256();    /* SHA256 + RSA */
 153                 break;
 154 #endif
 155 #ifdef HAVE_EVP_SHA512
 156         case DST_ALG_RSASHA512:
 157                 type = EVP_sha512();
 158                 break;
 159 #endif
 160         default:
 161                 INSIST(0);
 162         }
 163
 164         if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
 165                 EVP_MD_CTX_destroy(evp_md_ctx);
 166                 return (dst__openssl_toresult3(dctx->category,
 167                                                "EVP_DigestInit_ex",
 168                                                ISC_R_FAILURE));
 169         }
 170         dctx->ctxdata.evp_md_ctx = evp_md_ctx;
 171 #else
 172         switch (dctx->key->key_alg) {
 173         case DST_ALG_RSAMD5:
 174                 {
 175                         isc_md5_t *md5ctx;
 176
 177                         md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
 178                         if (md5ctx == NULL)
 179                                 return (ISC_R_NOMEMORY);
 180                         isc_md5_init(md5ctx);
 181                         dctx->ctxdata.md5ctx = md5ctx;
 182                 }
 183                 break;
 184         case DST_ALG_RSASHA1:
 185         case DST_ALG_NSEC3RSASHA1:
 186                 {
 187                         isc_sha1_t *sha1ctx;
 188
 189                         sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
 190                         if (sha1ctx == NULL)
 191                                 return (ISC_R_NOMEMORY);
 192                         isc_sha1_init(sha1ctx);
 193                         dctx->ctxdata.sha1ctx = sha1ctx;
 194                 }
 195                 break;
 196         case DST_ALG_RSASHA256:
 197                 {
 198                         isc_sha256_t *sha256ctx;
 199
 200                         sha256ctx = isc_mem_get(dctx->mctx,
 201                                                 sizeof(isc_sha256_t));
 202                         if (sha256ctx == NULL)
 203                                 return (ISC_R_NOMEMORY);
 204                         isc_sha256_init(sha256ctx);
 205                         dctx->ctxdata.sha256ctx = sha256ctx;
 206                 }
 207                 break;
 208         case DST_ALG_RSASHA512:
 209                 {
 210                         isc_sha512_t *sha512ctx;
 211
 212                         sha512ctx = isc_mem_get(dctx->mctx,
 213                                                 sizeof(isc_sha512_t));
 214                         if (sha512ctx == NULL)
 215                                 return (ISC_R_NOMEMORY);
 216                         isc_sha512_init(sha512ctx);
 217                         dctx->ctxdata.sha512ctx = sha512ctx;
 218                 }
 219                 break;
 220         default:
 221                 INSIST(0);
 222         }
 223 #endif
 224
 225         return (ISC_R_SUCCESS);
 226 }
 227
 228 static void
 229 opensslrsa_destroyctx(dst_context_t *dctx) {
 230 #if USE_EVP
 231         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 232 #endif
 233
 234         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 235                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 236                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 237                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 238                 dctx->key->key_alg == DST_ALG_RSASHA512);
 239
 240 #if USE_EVP
 241         if (evp_md_ctx != NULL) {
 242                 EVP_MD_CTX_destroy(evp_md_ctx);
 243                 dctx->ctxdata.evp_md_ctx = NULL;
 244         }
 245 #else
 246         switch (dctx->key->key_alg) {
 247         case DST_ALG_RSAMD5:
 248                 {
 249                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 250
 251                         if (md5ctx != NULL) {
 252                                 isc_md5_invalidate(md5ctx);
 253                                 isc_mem_put(dctx->mctx, md5ctx,
 254                                             sizeof(isc_md5_t));
 255                                 dctx->ctxdata.md5ctx = NULL;
 256                         }
 257                 }
 258                 break;
 259         case DST_ALG_RSASHA1:
 260         case DST_ALG_NSEC3RSASHA1:
 261                 {
 262                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 263
 264                         if (sha1ctx != NULL) {
 265                                 isc_sha1_invalidate(sha1ctx);
 266                                 isc_mem_put(dctx->mctx, sha1ctx,
 267                                             sizeof(isc_sha1_t));
 268                                 dctx->ctxdata.sha1ctx = NULL;
 269                         }
 270                 }
 271                 break;
 272         case DST_ALG_RSASHA256:
 273                 {
 274                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 275
 276                         if (sha256ctx != NULL) {
 277                                 isc_sha256_invalidate(sha256ctx);
 278                                 isc_mem_put(dctx->mctx, sha256ctx,
 279                                             sizeof(isc_sha256_t));
 280                                 dctx->ctxdata.sha256ctx = NULL;
 281                         }
 282                 }
 283                 break;
 284         case DST_ALG_RSASHA512:
 285                 {
 286                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 287
 288                         if (sha512ctx != NULL) {
 289                                 isc_sha512_invalidate(sha512ctx);
 290                                 isc_mem_put(dctx->mctx, sha512ctx,
 291                                             sizeof(isc_sha512_t));
 292                                 dctx->ctxdata.sha512ctx = NULL;
 293                         }
 294                 }
 295                 break;
 296         default:
 297                 INSIST(0);
 298         }
 299 #endif
 300 }
 301
 302 static isc_result_t
 303 opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
 304 #if USE_EVP
 305         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 306 #endif
 307
 308         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 309                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 310                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 311                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 312                 dctx->key->key_alg == DST_ALG_RSASHA512);
 313
 314 #if USE_EVP
 315         if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
 316                 return (dst__openssl_toresult3(dctx->category,
 317                                                "EVP_DigestUpdate",
 318                                                ISC_R_FAILURE));
 319         }
 320 #else
 321         switch (dctx->key->key_alg) {
 322         case DST_ALG_RSAMD5:
 323                 {
 324                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 325
 326                         isc_md5_update(md5ctx, data->base, data->length);
 327                 }
 328                 break;
 329         case DST_ALG_RSASHA1:
 330         case DST_ALG_NSEC3RSASHA1:
 331                 {
 332                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 333
 334                         isc_sha1_update(sha1ctx, data->base, data->length);
 335                 }
 336                 break;
 337         case DST_ALG_RSASHA256:
 338                 {
 339                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 340
 341                         isc_sha256_update(sha256ctx, data->base, data->length);
 342                 }
 343                 break;
 344         case DST_ALG_RSASHA512:
 345                 {
 346                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 347
 348                         isc_sha512_update(sha512ctx, data->base, data->length);
 349                 }
 350                 break;
 351         default:
 352                 INSIST(0);
 353         }
 354 #endif
 355         return (ISC_R_SUCCESS);
 356 }
 357
 358 #if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
 359 /*
 360  * Digest prefixes from RFC 5702.
 361  */
 362 static unsigned char sha256_prefix[] =
 363          { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
 364            0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
 365 static unsigned char sha512_prefix[] =
 366          { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
 367            0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
 368 #define PREFIXLEN sizeof(sha512_prefix)
 369 #else
 370 #define PREFIXLEN 0
 371 #endif
 372
 373 static isc_result_t
 374 opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 375         dst_key_t *key = dctx->key;
 376         isc_region_t r;
 377         unsigned int siglen = 0;
 378 #if USE_EVP
 379         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 380         EVP_PKEY *pkey = key->keydata.pkey;
 381 #else
 382         RSA *rsa = key->keydata.rsa;
 383         /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
 384         unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
 385         int status;
 386         int type = 0;
 387         unsigned int digestlen = 0;
 388 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 389         unsigned int prefixlen = 0;
 390         const unsigned char *prefix = NULL;
 391 #endif
 392 #endif
 393
 394         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 395                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 396                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 397                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 398                 dctx->key->key_alg == DST_ALG_RSASHA512);
 399
 400         isc_buffer_availableregion(sig, &r);
 401
 402 #if USE_EVP
 403         if (r.length < (unsigned int) EVP_PKEY_size(pkey))
 404                 return (ISC_R_NOSPACE);
 405
 406         if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
 407                 return (dst__openssl_toresult3(dctx->category,
 408                                                "EVP_SignFinal",
 409                                                ISC_R_FAILURE));
 410         }
 411 #else
 412         if (r.length < (unsigned int) RSA_size(rsa))
 413                 return (ISC_R_NOSPACE);
 414
 415         switch (dctx->key->key_alg) {
 416         case DST_ALG_RSAMD5:
 417                 {
 418                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 419
 420                         isc_md5_final(md5ctx, digest);
 421                         type = NID_md5;
 422                         digestlen = ISC_MD5_DIGESTLENGTH;
 423                 }
 424                 break;
 425         case DST_ALG_RSASHA1:
 426         case DST_ALG_NSEC3RSASHA1:
 427                 {
 428                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 429
 430                         isc_sha1_final(sha1ctx, digest);
 431                         type = NID_sha1;
 432                         digestlen = ISC_SHA1_DIGESTLENGTH;
 433                 }
 434                 break;
 435         case DST_ALG_RSASHA256:
 436                 {
 437                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 438
 439                         isc_sha256_final(digest, sha256ctx);
 440                         digestlen = ISC_SHA256_DIGESTLENGTH;
 441 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 442                         prefix = sha256_prefix;
 443                         prefixlen = sizeof(sha256_prefix);
 444 #else
 445                         type = NID_sha256;
 446 #endif
 447                 }
 448                 break;
 449         case DST_ALG_RSASHA512:
 450                 {
 451                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 452
 453                         isc_sha512_final(digest, sha512ctx);
 454                         digestlen = ISC_SHA512_DIGESTLENGTH;
 455 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 456                         prefix = sha512_prefix;
 457                         prefixlen = sizeof(sha512_prefix);
 458 #else
 459                         type = NID_sha512;
 460 #endif
 461                 }
 462                 break;
 463         default:
 464                 INSIST(0);
 465         }
 466
 467 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 468         switch (dctx->key->key_alg) {
 469         case DST_ALG_RSAMD5:
 470         case DST_ALG_RSASHA1:
 471         case DST_ALG_NSEC3RSASHA1:
 472                 INSIST(type != 0);
 473                 status = RSA_sign(type, digest, digestlen, r.base,
 474                                   &siglen, rsa);
 475                 break;
 476
 477         case DST_ALG_RSASHA256:
 478         case DST_ALG_RSASHA512:
 479                 INSIST(prefix != NULL);
 480                 INSIST(prefixlen != 0);
 481                 INSIST(prefixlen + digestlen <= sizeof(digest));
 482
 483                 memmove(digest + prefixlen, digest, digestlen);
 484                 memmove(digest, prefix, prefixlen);
 485                 status = RSA_private_encrypt(digestlen + prefixlen,
 486                                              digest, r.base, rsa,
 487                                              RSA_PKCS1_PADDING);
 488                 if (status < 0)
 489                         status = 0;
 490                 else
 491                         siglen = status;
 492                 break;
 493
 494         default:
 495                 INSIST(0);
 496         }
 497 #else
 498         INSIST(type != 0);
 499         status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
 500 #endif
 501         if (status == 0)
 502                 return (dst__openssl_toresult3(dctx->category,
 503                                                "RSA_sign",
 504                                                DST_R_OPENSSLFAILURE));
 505 #endif
 506
 507         isc_buffer_add(sig, siglen);
 508
 509         return (ISC_R_SUCCESS);
 510 }
 511
 512 static isc_result_t
 513 opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
 514         dst_key_t *key = dctx->key;
 515         int status = 0;
 516 #if USE_EVP
 517         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 518         EVP_PKEY *pkey = key->keydata.pkey;
 519         RSA *rsa;
 520         int bits;
 521 #else
 522         /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
 523         unsigned char digest[ISC_SHA512_DIGESTLENGTH];
 524         int type = 0;
 525         unsigned int digestlen = 0;
 526         RSA *rsa = key->keydata.rsa;
 527 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 528         unsigned int prefixlen = 0;
 529         const unsigned char *prefix = NULL;
 530 #endif
 531 #endif
 532
 533         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 534                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 535                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 536                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 537                 dctx->key->key_alg == DST_ALG_RSASHA512);
 538
 539 #if USE_EVP
 540         rsa = EVP_PKEY_get1_RSA(pkey);
 541         if (rsa == NULL)
 542                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 543         bits = BN_num_bits(rsa->e);
 544         RSA_free(rsa);
 545         if (bits > maxbits && maxbits != 0)
 546                 return (DST_R_VERIFYFAILURE);
 547
 548         status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
 549         switch (status) {
 550         case 1:
 551                 return (ISC_R_SUCCESS);
 552         case 0:
 553                 return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
 554         default:
 555                 return (dst__openssl_toresult3(dctx->category,
 556                                                "EVP_VerifyFinal",
 557                                                DST_R_VERIFYFAILURE));
 558         }
 559 #else
 560         if (BN_num_bits(rsa->e) > maxbits && maxbits != 0)
 561                 return (DST_R_VERIFYFAILURE);
 562
 563         switch (dctx->key->key_alg) {
 564         case DST_ALG_RSAMD5:
 565                 {
 566                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 567
 568                         isc_md5_final(md5ctx, digest);
 569                         type = NID_md5;
 570                         digestlen = ISC_MD5_DIGESTLENGTH;
 571                 }
 572                 break;
 573         case DST_ALG_RSASHA1:
 574         case DST_ALG_NSEC3RSASHA1:
 575                 {
 576                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 577
 578                         isc_sha1_final(sha1ctx, digest);
 579                         type = NID_sha1;
 580                         digestlen = ISC_SHA1_DIGESTLENGTH;
 581                 }
 582                 break;
 583         case DST_ALG_RSASHA256:
 584                 {
 585                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 586
 587                         isc_sha256_final(digest, sha256ctx);
 588                         digestlen = ISC_SHA256_DIGESTLENGTH;
 589 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 590                         prefix = sha256_prefix;
 591                         prefixlen = sizeof(sha256_prefix);
 592 #else
 593                         type = NID_sha256;
 594 #endif
 595                 }
 596                 break;
 597         case DST_ALG_RSASHA512:
 598                 {
 599                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 600
 601                         isc_sha512_final(digest, sha512ctx);
 602                         digestlen = ISC_SHA512_DIGESTLENGTH;
 603 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 604                         prefix = sha512_prefix;
 605                         prefixlen = sizeof(sha512_prefix);
 606 #else
 607                         type = NID_sha512;
 608 #endif
 609                 }
 610                 break;
 611         default:
 612                 INSIST(0);
 613         }
 614
 615         if (sig->length != (unsigned int) RSA_size(rsa))
 616                 return (DST_R_VERIFYFAILURE);
 617
 618 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 619         switch (dctx->key->key_alg) {
 620         case DST_ALG_RSAMD5:
 621         case DST_ALG_RSASHA1:
 622         case DST_ALG_NSEC3RSASHA1:
 623                 INSIST(type != 0);
 624                 status = RSA_verify(type, digest, digestlen, sig->base,
 625                                     RSA_size(rsa), rsa);
 626                 break;
 627
 628         case DST_ALG_RSASHA256:
 629         case DST_ALG_RSASHA512:
 630                 {
 631                         /*
 632                          * 1024 is big enough for all valid RSA bit sizes
 633                          * for use with DNSSEC.
 634                          */
 635                         unsigned char original[PREFIXLEN + 1024];
 636
 637                         INSIST(prefix != NULL);
 638                         INSIST(prefixlen != 0U);
 639
 640                         if (RSA_size(rsa) > (int)sizeof(original))
 641                                 return (DST_R_VERIFYFAILURE);
 642
 643                         status = RSA_public_decrypt(sig->length, sig->base,
 644                                                     original, rsa,
 645                                                     RSA_PKCS1_PADDING);
 646                         if (status <= 0)
 647                                 return (dst__openssl_toresult3(
 648                                                 dctx->category,
 649                                                 "RSA_public_decrypt",
 650                                                 DST_R_VERIFYFAILURE));
 651                         if (status != (int)(prefixlen + digestlen))
 652                                 return (DST_R_VERIFYFAILURE);
 653                         if (memcmp(original, prefix, prefixlen))
 654                                 return (DST_R_VERIFYFAILURE);
 655                         if (memcmp(original + prefixlen, digest, digestlen))
 656                                 return (DST_R_VERIFYFAILURE);
 657                         status = 1;
 658                 }
 659                 break;
 660
 661         default:
 662                 INSIST(0);
 663         }
 664 #else
 665         INSIST(type != 0);
 666         status = RSA_verify(type, digest, digestlen, sig->base,
 667                              RSA_size(rsa), rsa);
 668 #endif
 669         if (status != 1)
 670                 return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
 671         return (ISC_R_SUCCESS);
 672 #endif
 673 }
 674
 675 static isc_result_t
 676 opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 677         return (opensslrsa_verify2(dctx, 0, sig));
 678 }
 679
 680 static isc_boolean_t
 681 opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
 682         int status;
 683         RSA *rsa1 = NULL, *rsa2 = NULL;
 684 #if USE_EVP
 685         EVP_PKEY *pkey1, *pkey2;
 686 #endif
 687
 688 #if USE_EVP
 689         pkey1 = key1->keydata.pkey;
 690         pkey2 = key2->keydata.pkey;
 691         /*
 692          * The pkey reference will keep these around after
 693          * the RSA_free() call.
 694          */
 695         if (pkey1 != NULL) {
 696                 rsa1 = EVP_PKEY_get1_RSA(pkey1);
 697                 RSA_free(rsa1);
 698         }
 699         if (pkey2 != NULL) {
 700                 rsa2 = EVP_PKEY_get1_RSA(pkey2);
 701                 RSA_free(rsa2);
 702         }
 703 #else
 704         rsa1 = key1->keydata.rsa;
 705         rsa2 = key2->keydata.rsa;
 706 #endif
 707
 708         if (rsa1 == NULL && rsa2 == NULL)
 709                 return (ISC_TRUE);
 710         else if (rsa1 == NULL || rsa2 == NULL)
 711                 return (ISC_FALSE);
 712
 713         status = BN_cmp(rsa1->n, rsa2->n) ||
 714                  BN_cmp(rsa1->e, rsa2->e);
 715
 716         if (status != 0)
 717                 return (ISC_FALSE);
 718
 719 #if USE_EVP
 720         if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
 721             (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
 722                 if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
 723                     (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
 724                         return (ISC_FALSE);
 725                 /*
 726                  * Can't compare private parameters, BTW does it make sense?
 727                  */
 728                 return (ISC_TRUE);
 729         }
 730 #endif
 731
 732         if (rsa1->d != NULL || rsa2->d != NULL) {
 733                 if (rsa1->d == NULL || rsa2->d == NULL)
 734                         return (ISC_FALSE);
 735                 status = BN_cmp(rsa1->d, rsa2->d) ||
 736                          BN_cmp(rsa1->p, rsa2->p) ||
 737                          BN_cmp(rsa1->q, rsa2->q);
 738
 739                 if (status != 0)
 740                         return (ISC_FALSE);
 741         }
 742         return (ISC_TRUE);
 743 }
 744
 745 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 746 static int
 747 progress_cb(int p, int n, BN_GENCB *cb)
 748 {
 749         union {
 750                 void *dptr;
 751                 void (*fptr)(int);
 752         } u;
 753
 754         UNUSED(n);
 755
 756         u.dptr = cb->arg;
 757         if (u.fptr != NULL)
 758                 u.fptr(p);
 759         return (1);
 760 }
 761 #endif
 762
 763 static isc_result_t
 764 opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 765 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 766         isc_result_t ret = DST_R_OPENSSLFAILURE;
 767         BN_GENCB cb;
 768         union {
 769                 void *dptr;
 770                 void (*fptr)(int);
 771         } u;
 772         RSA *rsa = RSA_new();
 773         BIGNUM *e = BN_new();
 774 #if USE_EVP
 775         EVP_PKEY *pkey = EVP_PKEY_new();
 776 #endif
 777
 778         if (rsa == NULL || e == NULL)
 779                 goto err;
 780 #if USE_EVP
 781         if (pkey == NULL)
 782                 goto err;
 783         if (!EVP_PKEY_set1_RSA(pkey, rsa))
 784                 goto err;
 785 #endif
 786
 787         if (exp == 0) {
 788                 /* RSA_F4 0x10001 */
 789                 BN_set_bit(e, 0);
 790                 BN_set_bit(e, 16);
 791         } else {
 792                 /* (phased-out) F5 0x100000001 */
 793                 BN_set_bit(e, 0);
 794                 BN_set_bit(e, 32);
 795         }
 796
 797         if (callback == NULL) {
 798                 BN_GENCB_set_old(&cb, NULL, NULL);
 799         } else {
 800                 u.fptr = callback;
 801                 BN_GENCB_set(&cb, &progress_cb, u.dptr);
 802         }
 803
 804         if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
 805                 BN_free(e);
 806                 SET_FLAGS(rsa);
 807 #if USE_EVP
 808                 key->keydata.pkey = pkey;
 809
 810                 RSA_free(rsa);
 811 #else
 812                 key->keydata.rsa = rsa;
 813 #endif
 814                 return (ISC_R_SUCCESS);
 815         }
 816         ret = dst__openssl_toresult2("RSA_generate_key_ex",
 817                                      DST_R_OPENSSLFAILURE);
 818
 819 err:
 820 #if USE_EVP
 821         if (pkey != NULL)
 822                 EVP_PKEY_free(pkey);
 823 #endif
 824         if (e != NULL)
 825                 BN_free(e);
 826         if (rsa != NULL)
 827                 RSA_free(rsa);
 828         return (dst__openssl_toresult(ret));
 829 #else
 830         RSA *rsa;
 831         unsigned long e;
 832 #if USE_EVP
 833         EVP_PKEY *pkey = EVP_PKEY_new();
 834
 835         UNUSED(callback);
 836
 837         if (pkey == NULL)
 838                 return (ISC_R_NOMEMORY);
 839 #else
 840         UNUSED(callback);
 841 #endif
 842
 843         if (exp == 0)
 844                e = RSA_F4;
 845         else
 846                e = 0x40000003;
 847         rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
 848         if (rsa == NULL) {
 849 #if USE_EVP
 850                 EVP_PKEY_free(pkey);
 851 #endif
 852                 return (dst__openssl_toresult2("RSA_generate_key",
 853                                                DST_R_OPENSSLFAILURE));
 854         }
 855         SET_FLAGS(rsa);
 856 #if USE_EVP
 857         if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
 858                 EVP_PKEY_free(pkey);
 859                 RSA_free(rsa);
 860                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 861         }
 862         key->keydata.pkey = pkey;
 863         RSA_free(rsa);
 864 #else
 865         key->keydata.rsa = rsa;
 866 #endif
 867
 868         return (ISC_R_SUCCESS);
 869 #endif
 870 }
 871
 872 static isc_boolean_t
 873 opensslrsa_isprivate(const dst_key_t *key) {
 874 #if USE_EVP
 875         RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
 876         INSIST(rsa != NULL);
 877         RSA_free(rsa);
 878         /* key->keydata.pkey still has a reference so rsa is still valid. */
 879 #else
 880         RSA *rsa = key->keydata.rsa;
 881 #endif
 882         if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
 883                 return (ISC_TRUE);
 884         return (ISC_TF(rsa != NULL && rsa->d != NULL));
 885 }
 886
 887 static void
 888 opensslrsa_destroy(dst_key_t *key) {
 889 #if USE_EVP
 890         EVP_PKEY *pkey = key->keydata.pkey;
 891         EVP_PKEY_free(pkey);
 892         key->keydata.pkey = NULL;
 893 #else
 894         RSA *rsa = key->keydata.rsa;
 895         RSA_free(rsa);
 896         key->keydata.rsa = NULL;
 897 #endif
 898 }
 899
 900
 901 static isc_result_t
 902 opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
 903         isc_region_t r;
 904         unsigned int e_bytes;
 905         unsigned int mod_bytes;
 906         isc_result_t ret;
 907         RSA *rsa;
 908 #if USE_EVP
 909         EVP_PKEY *pkey;
 910 #endif
 911
 912 #if USE_EVP
 913         REQUIRE(key->keydata.pkey != NULL);
 914 #else
 915         REQUIRE(key->keydata.rsa != NULL);
 916 #endif
 917
 918 #if USE_EVP
 919         pkey = key->keydata.pkey;
 920         rsa = EVP_PKEY_get1_RSA(pkey);
 921         if (rsa == NULL)
 922                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 923 #else
 924         rsa = key->keydata.rsa;
 925 #endif
 926
 927         isc_buffer_availableregion(data, &r);
 928
 929         e_bytes = BN_num_bytes(rsa->e);
 930         mod_bytes = BN_num_bytes(rsa->n);
 931
 932         if (e_bytes < 256) {    /*%< key exponent is <= 2040 bits */
 933                 if (r.length < 1)
 934                         DST_RET(ISC_R_NOSPACE);
 935                 isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
 936                 isc_region_consume(&r, 1);
 937         } else {
 938                 if (r.length < 3)
 939                         DST_RET(ISC_R_NOSPACE);
 940                 isc_buffer_putuint8(data, 0);
 941                 isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
 942                 isc_region_consume(&r, 3);
 943         }
 944
 945         if (r.length < e_bytes + mod_bytes)
 946                 DST_RET(ISC_R_NOSPACE);
 947
 948         BN_bn2bin(rsa->e, r.base);
 949         isc_region_consume(&r, e_bytes);
 950         BN_bn2bin(rsa->n, r.base);
 951
 952         isc_buffer_add(data, e_bytes + mod_bytes);
 953
 954         ret = ISC_R_SUCCESS;
 955  err:
 956 #if USE_EVP
 957         if (rsa != NULL)
 958                 RSA_free(rsa);
 959 #endif
 960         return (ret);
 961 }
 962
 963 static isc_result_t
 964 opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 965         RSA *rsa;
 966         isc_region_t r;
 967         unsigned int e_bytes;
 968         unsigned int length;
 969 #if USE_EVP
 970         EVP_PKEY *pkey;
 971 #endif
 972
 973         isc_buffer_remainingregion(data, &r);
 974         if (r.length == 0)
 975                 return (ISC_R_SUCCESS);
 976         length = r.length;
 977
 978         rsa = RSA_new();
 979         if (rsa == NULL)
 980                 return (dst__openssl_toresult(ISC_R_NOMEMORY));
 981         SET_FLAGS(rsa);
 982
 983         if (r.length < 1) {
 984                 RSA_free(rsa);
 985                 return (DST_R_INVALIDPUBLICKEY);
 986         }
 987         e_bytes = *r.base;
 988         isc_region_consume(&r, 1);
 989
 990         if (e_bytes == 0) {
 991                 if (r.length < 2) {
 992                         RSA_free(rsa);
 993                         return (DST_R_INVALIDPUBLICKEY);
 994                 }
 995                 e_bytes = (*r.base) << 8;
 996                 isc_region_consume(&r, 1);
 997                 e_bytes += *r.base;
 998                 isc_region_consume(&r, 1);
 999         }
1000
1001         if (r.length < e_bytes) {
1002                 RSA_free(rsa);
1003                 return (DST_R_INVALIDPUBLICKEY);
1004         }
1005         rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
1006         isc_region_consume(&r, e_bytes);
1007
1008         rsa->n = BN_bin2bn(r.base, r.length, NULL);
1009
1010         key->key_size = BN_num_bits(rsa->n);
1011
1012         isc_buffer_forward(data, length);
1013
1014 #if USE_EVP
1015         pkey = EVP_PKEY_new();
1016         if (pkey == NULL) {
1017                 RSA_free(rsa);
1018                 return (ISC_R_NOMEMORY);
1019         }
1020         if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
1021                 EVP_PKEY_free(pkey);
1022                 RSA_free(rsa);
1023                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1024         }
1025         key->keydata.pkey = pkey;
1026         RSA_free(rsa);
1027 #else
1028         key->keydata.rsa = rsa;
1029 #endif
1030
1031         return (ISC_R_SUCCESS);
1032 }
1033
1034 static isc_result_t
1035 opensslrsa_tofile(const dst_key_t *key, const char *directory) {
1036         int i;
1037         RSA *rsa;
1038         dst_private_t priv;
1039         unsigned char *bufs[8];
1040         isc_result_t result;
1041
1042 #if USE_EVP
1043         if (key->keydata.pkey == NULL)
1044                 return (DST_R_NULLKEY);
1045         rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
1046         if (rsa == NULL)
1047                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1048 #else
1049         if (key->keydata.rsa == NULL)
1050                 return (DST_R_NULLKEY);
1051         rsa = key->keydata.rsa;
1052 #endif
1053         memset(bufs, 0, sizeof(bufs));
1054
1055         if (key->external) {
1056                 priv.nelements = 0;
1057                 result = dst__privstruct_writefile(key, &priv, directory);
1058                 goto fail;
1059         }
1060
1061         for (i = 0; i < 8; i++) {
1062                 bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
1063                 if (bufs[i] == NULL) {
1064                         result = ISC_R_NOMEMORY;
1065                         goto fail;
1066                 }
1067         }
1068
1069         i = 0;
1070
1071         priv.elements[i].tag = TAG_RSA_MODULUS;
1072         priv.elements[i].length = BN_num_bytes(rsa->n);
1073         BN_bn2bin(rsa->n, bufs[i]);
1074         priv.elements[i].data = bufs[i];
1075         i++;
1076
1077         priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
1078         priv.elements[i].length = BN_num_bytes(rsa->e);
1079         BN_bn2bin(rsa->e, bufs[i]);
1080         priv.elements[i].data = bufs[i];
1081         i++;
1082
1083         if (rsa->d != NULL) {
1084                 priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
1085                 priv.elements[i].length = BN_num_bytes(rsa->d);
1086                 BN_bn2bin(rsa->d, bufs[i]);
1087                 priv.elements[i].data = bufs[i];
1088                 i++;
1089         }
1090
1091         if (rsa->p != NULL) {
1092                 priv.elements[i].tag = TAG_RSA_PRIME1;
1093                 priv.elements[i].length = BN_num_bytes(rsa->p);
1094                 BN_bn2bin(rsa->p, bufs[i]);
1095                 priv.elements[i].data = bufs[i];
1096                 i++;
1097         }
1098
1099         if (rsa->q != NULL) {
1100                 priv.elements[i].tag = TAG_RSA_PRIME2;
1101                 priv.elements[i].length = BN_num_bytes(rsa->q);
1102                 BN_bn2bin(rsa->q, bufs[i]);
1103                 priv.elements[i].data = bufs[i];
1104                 i++;
1105         }
1106
1107         if (rsa->dmp1 != NULL) {
1108                 priv.elements[i].tag = TAG_RSA_EXPONENT1;
1109                 priv.elements[i].length = BN_num_bytes(rsa->dmp1);
1110                 BN_bn2bin(rsa->dmp1, bufs[i]);
1111                 priv.elements[i].data = bufs[i];
1112                 i++;
1113         }
1114
1115         if (rsa->dmq1 != NULL) {
1116                 priv.elements[i].tag = TAG_RSA_EXPONENT2;
1117                 priv.elements[i].length = BN_num_bytes(rsa->dmq1);
1118                 BN_bn2bin(rsa->dmq1, bufs[i]);
1119                 priv.elements[i].data = bufs[i];
1120                 i++;
1121         }
1122
1123         if (rsa->iqmp != NULL) {
1124                 priv.elements[i].tag = TAG_RSA_COEFFICIENT;
1125                 priv.elements[i].length = BN_num_bytes(rsa->iqmp);
1126                 BN_bn2bin(rsa->iqmp, bufs[i]);
1127                 priv.elements[i].data = bufs[i];
1128                 i++;
1129         }
1130
1131         if (key->engine != NULL) {
1132                 priv.elements[i].tag = TAG_RSA_ENGINE;
1133                 priv.elements[i].length = strlen(key->engine) + 1;
1134                 priv.elements[i].data = (unsigned char *)key->engine;
1135                 i++;
1136         }
1137
1138         if (key->label != NULL) {
1139                 priv.elements[i].tag = TAG_RSA_LABEL;
1140                 priv.elements[i].length = strlen(key->label) + 1;
1141                 priv.elements[i].data = (unsigned char *)key->label;
1142                 i++;
1143         }
1144
1145
1146         priv.nelements = i;
1147         result = dst__privstruct_writefile(key, &priv, directory);
1148  fail:
1149 #if USE_EVP
1150         RSA_free(rsa);
1151 #endif
1152         for (i = 0; i < 8; i++) {
1153                 if (bufs[i] == NULL)
1154                         break;
1155                 isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
1156         }
1157         return (result);
1158 }
1159
1160 static isc_result_t
1161 rsa_check(RSA *rsa, RSA *pub)
1162 {
1163         /* Public parameters should be the same but if they are not set
1164          * copy them from the public key. */
1165         if (pub != NULL) {
1166                 if (rsa->n != NULL) {
1167                         if (BN_cmp(rsa->n, pub->n) != 0)
1168                                 return (DST_R_INVALIDPRIVATEKEY);
1169                 } else {
1170                         rsa->n = pub->n;
1171                         pub->n = NULL;
1172                 }
1173                 if (rsa->e != NULL) {
1174                         if (BN_cmp(rsa->e, pub->e) != 0)
1175                                 return (DST_R_INVALIDPRIVATEKEY);
1176                 } else {
1177                         rsa->e = pub->e;
1178                         pub->e = NULL;
1179                 }
1180         }
1181         if (rsa->n == NULL || rsa->e == NULL)
1182                 return (DST_R_INVALIDPRIVATEKEY);
1183         return (ISC_R_SUCCESS);
1184 }
1185
1186 static isc_result_t
1187 opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1188         dst_private_t priv;
1189         isc_result_t ret;
1190         int i;
1191         RSA *rsa = NULL, *pubrsa = NULL;
1192 #ifdef USE_ENGINE
1193         ENGINE *e = NULL;
1194 #endif
1195         isc_mem_t *mctx = key->mctx;
1196         const char *engine = NULL, *label = NULL;
1197 #if defined(USE_ENGINE) || USE_EVP
1198         EVP_PKEY *pkey = NULL;
1199 #endif
1200
1201 #if USE_EVP
1202         if (pub != NULL && pub->keydata.pkey != NULL)
1203                 pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
1204 #else
1205         if (pub != NULL && pub->keydata.rsa != NULL) {
1206                 pubrsa = pub->keydata.rsa;
1207                 pub->keydata.rsa = NULL;
1208         }
1209 #endif
1210
1211         /* read private key file */
1212         ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1213         if (ret != ISC_R_SUCCESS)
1214                 goto err;
1215
1216         if (key->external && priv.nelements != 0)
1217                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1218
1219         for (i = 0; i < priv.nelements; i++) {
1220                 switch (priv.elements[i].tag) {
1221                 case TAG_RSA_ENGINE:
1222                         engine = (char *)priv.elements[i].data;
1223                         break;
1224                 case TAG_RSA_LABEL:
1225                         label = (char *)priv.elements[i].data;
1226                         break;
1227                 default:
1228                         break;
1229                 }
1230         }
1231
1232         /*
1233          * Is this key is stored in a HSM?
1234          * See if we can fetch it.
1235          */
1236         if (label != NULL) {
1237 #ifdef USE_ENGINE
1238                 if (engine == NULL)
1239                         DST_RET(DST_R_NOENGINE);
1240                 e = dst__openssl_getengine(engine);
1241                 if (e == NULL)
1242                         DST_RET(DST_R_NOENGINE);
1243                 pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1244                 if (pkey == NULL)
1245                         DST_RET(dst__openssl_toresult2(
1246                                         "ENGINE_load_private_key",
1247                                         ISC_R_NOTFOUND));
1248                 key->engine = isc_mem_strdup(key->mctx, engine);
1249                 if (key->engine == NULL)
1250                         DST_RET(ISC_R_NOMEMORY);
1251                 key->label = isc_mem_strdup(key->mctx, label);
1252                 if (key->label == NULL)
1253                         DST_RET(ISC_R_NOMEMORY);
1254                 rsa = EVP_PKEY_get1_RSA(pkey);
1255                 if (rsa == NULL)
1256                         DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1257                 if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1258                         DST_RET(DST_R_INVALIDPRIVATEKEY);
1259                 if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1260                         DST_RET(ISC_R_RANGE);
1261                 if (pubrsa != NULL)
1262                         RSA_free(pubrsa);
1263                 key->key_size = EVP_PKEY_bits(pkey);
1264 #if USE_EVP
1265                 key->keydata.pkey = pkey;
1266                 RSA_free(rsa);
1267 #else
1268                 key->keydata.rsa = rsa;
1269                 EVP_PKEY_free(pkey);
1270 #endif
1271                 dst__privstruct_free(&priv, mctx);
1272                 memset(&priv, 0, sizeof(priv));
1273                 return (ISC_R_SUCCESS);
1274 #else
1275                 DST_RET(DST_R_NOENGINE);
1276 #endif
1277         }
1278
1279         rsa = RSA_new();
1280         if (rsa == NULL)
1281                 DST_RET(ISC_R_NOMEMORY);
1282         SET_FLAGS(rsa);
1283
1284 #if USE_EVP
1285         pkey = EVP_PKEY_new();
1286         if (pkey == NULL)
1287                 DST_RET(ISC_R_NOMEMORY);
1288         if (!EVP_PKEY_set1_RSA(pkey, rsa))
1289                 DST_RET(ISC_R_FAILURE);
1290         key->keydata.pkey = pkey;
1291 #else
1292         key->keydata.rsa = rsa;
1293 #endif
1294
1295         for (i = 0; i < priv.nelements; i++) {
1296                 BIGNUM *bn;
1297                 switch (priv.elements[i].tag) {
1298                 case TAG_RSA_ENGINE:
1299                         continue;
1300                 case TAG_RSA_LABEL:
1301                         continue;
1302                 case TAG_RSA_PIN:
1303                         continue;
1304                 default:
1305                         bn = BN_bin2bn(priv.elements[i].data,
1306                                        priv.elements[i].length, NULL);
1307                         if (bn == NULL)
1308                                 DST_RET(ISC_R_NOMEMORY);
1309                 }
1310
1311                 switch (priv.elements[i].tag) {
1312                         case TAG_RSA_MODULUS:
1313                                 rsa->n = bn;
1314                                 break;
1315                         case TAG_RSA_PUBLICEXPONENT:
1316                                 rsa->e = bn;
1317                                 break;
1318                         case TAG_RSA_PRIVATEEXPONENT:
1319                                 rsa->d = bn;
1320                                 break;
1321                         case TAG_RSA_PRIME1:
1322                                 rsa->p = bn;
1323                                 break;
1324                         case TAG_RSA_PRIME2:
1325                                 rsa->q = bn;
1326                                 break;
1327                         case TAG_RSA_EXPONENT1:
1328                                 rsa->dmp1 = bn;
1329                                 break;
1330                         case TAG_RSA_EXPONENT2:
1331                                 rsa->dmq1 = bn;
1332                                 break;
1333                         case TAG_RSA_COEFFICIENT:
1334                                 rsa->iqmp = bn;
1335                                 break;
1336                 }
1337         }
1338         dst__privstruct_free(&priv, mctx);
1339         memset(&priv, 0, sizeof(priv));
1340
1341         if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1342                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1343         if (!key->external) {
1344                 if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1345                         DST_RET(ISC_R_RANGE);
1346         }
1347         key->key_size = BN_num_bits(rsa->n);
1348         if (pubrsa != NULL)
1349                 RSA_free(pubrsa);
1350 #if USE_EVP
1351         RSA_free(rsa);
1352 #endif
1353
1354         return (ISC_R_SUCCESS);
1355
1356  err:
1357 #if USE_EVP
1358         if (pkey != NULL)
1359                 EVP_PKEY_free(pkey);
1360 #endif
1361         if (rsa != NULL)
1362                 RSA_free(rsa);
1363         if (pubrsa != NULL)
1364                 RSA_free(pubrsa);
1365         key->keydata.generic = NULL;
1366         dst__privstruct_free(&priv, mctx);
1367         memset(&priv, 0, sizeof(priv));
1368         return (ret);
1369 }
1370
1371 static isc_result_t
1372 opensslrsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
1373                      const char *pin)
1374 {
1375 #ifdef USE_ENGINE
1376         ENGINE *e = NULL;
1377         isc_result_t ret;
1378         EVP_PKEY *pkey = NULL;
1379         RSA *rsa = NULL, *pubrsa = NULL;
1380         char *colon;
1381
1382         UNUSED(pin);
1383
1384         if (engine == NULL)
1385                 DST_RET(DST_R_NOENGINE);
1386         e = dst__openssl_getengine(engine);
1387         if (e == NULL)
1388                 DST_RET(DST_R_NOENGINE);
1389         pkey = ENGINE_load_public_key(e, label, NULL, NULL);
1390         if (pkey != NULL) {
1391                 pubrsa = EVP_PKEY_get1_RSA(pkey);
1392                 EVP_PKEY_free(pkey);
1393                 if (pubrsa == NULL)
1394                         DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1395         }
1396         pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1397         if (pkey == NULL)
1398                 DST_RET(dst__openssl_toresult2("ENGINE_load_private_key",
1399                                                ISC_R_NOTFOUND));
1400         if (engine != NULL) {
1401                 key->engine = isc_mem_strdup(key->mctx, engine);
1402                 if (key->engine == NULL)
1403                         DST_RET(ISC_R_NOMEMORY);
1404         } else {
1405                 key->engine = isc_mem_strdup(key->mctx, label);
1406                 if (key->engine == NULL)
1407                         DST_RET(ISC_R_NOMEMORY);
1408                 colon = strchr(key->engine, ':');
1409                 if (colon != NULL)
1410                         *colon = '\0';
1411         }
1412         key->label = isc_mem_strdup(key->mctx, label);
1413         if (key->label == NULL)
1414                 DST_RET(ISC_R_NOMEMORY);
1415         rsa = EVP_PKEY_get1_RSA(pkey);
1416         if (rsa == NULL)
1417                 DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1418         if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1419                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1420         if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1421                 DST_RET(ISC_R_RANGE);
1422         if (pubrsa != NULL)
1423                 RSA_free(pubrsa);
1424         key->key_size = EVP_PKEY_bits(pkey);
1425 #if USE_EVP
1426         key->keydata.pkey = pkey;
1427         RSA_free(rsa);
1428 #else
1429         key->keydata.rsa = rsa;
1430         EVP_PKEY_free(pkey);
1431 #endif
1432         return (ISC_R_SUCCESS);
1433
1434  err:
1435         if (rsa != NULL)
1436                 RSA_free(rsa);
1437         if (pubrsa != NULL)
1438                 RSA_free(pubrsa);
1439         if (pkey != NULL)
1440                 EVP_PKEY_free(pkey);
1441         return (ret);
1442 #else
1443         UNUSED(key);
1444         UNUSED(engine);
1445         UNUSED(label);
1446         UNUSED(pin);
1447         return(DST_R_NOENGINE);
1448 #endif
1449 }
1450
1451 static dst_func_t opensslrsa_functions = {
1452         opensslrsa_createctx,
1453         opensslrsa_destroyctx,
1454         opensslrsa_adddata,
1455         opensslrsa_sign,
1456         opensslrsa_verify,
1457         opensslrsa_verify2,
1458         NULL, /*%< computesecret */
1459         opensslrsa_compare,
1460         NULL, /*%< paramcompare */
1461         opensslrsa_generate,
1462         opensslrsa_isprivate,
1463         opensslrsa_destroy,
1464         opensslrsa_todns,
1465         opensslrsa_fromdns,
1466         opensslrsa_tofile,
1467         opensslrsa_parse,
1468         NULL, /*%< cleanup */
1469         opensslrsa_fromlabel,
1470         NULL, /*%< dump */
1471         NULL, /*%< restore */
1472 };
1473
1474 isc_result_t
1475 dst__opensslrsa_init(dst_func_t **funcp, unsigned char algorithm) {
1476         REQUIRE(funcp != NULL);
1477
1478         if (*funcp == NULL) {
1479                 switch (algorithm) {
1480                 case DST_ALG_RSASHA256:
1481 #if defined(HAVE_EVP_SHA256) || !USE_EVP
1482                         *funcp = &opensslrsa_functions;
1483 #endif
1484                         break;
1485                 case DST_ALG_RSASHA512:
1486 #if defined(HAVE_EVP_SHA512) || !USE_EVP
1487                         *funcp = &opensslrsa_functions;
1488 #endif
1489                         break;
1490                 default:
1491                         *funcp = &opensslrsa_functions;
1492                         break;
1493                 }
1494         }
1495         return (ISC_R_SUCCESS);
1496 }
1497
1498 #else /* OPENSSL */
1499
1500 #include <isc/util.h>
1501
1502 EMPTY_TRANSLATION_UNIT
1503
1504 #endif /* OPENSSL */
1505 /*! \file */