contrib/bind9/lib/dns/opensslrsa_link.c

   1 /*
   2  * Copyright (C) 2004-2009, 2011-2014  Internet Systems Consortium, Inc. ("ISC")
   3  * Copyright (C) 2000-2003  Internet Software Consortium.
   4  *
   5  * Permission to use, copy, modify, and/or distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
  10  * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
  11  * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
  12  * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  13  * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
  14  * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  15  * PERFORMANCE OF THIS SOFTWARE.
  16  */
  17
  18 /*
  19  * Principal Author: Brian Wellington
  20  * $Id$
  21  */
  22 #ifdef OPENSSL
  23 #include <config.h>
  24
  25 #ifndef USE_EVP
  26 #if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
  27 #define USE_EVP 0
  28 #else
  29 #define USE_EVP 1
  30 #endif
  31 #endif
  32
  33
  34 #include <isc/entropy.h>
  35 #include <isc/md5.h>
  36 #include <isc/sha1.h>
  37 #include <isc/sha2.h>
  38 #include <isc/mem.h>
  39 #include <isc/string.h>
  40 #include <isc/util.h>
  41
  42 #include <dst/result.h>
  43
  44 #include "dst_internal.h"
  45 #include "dst_openssl.h"
  46 #include "dst_parse.h"
  47
  48 #include <openssl/err.h>
  49 #include <openssl/objects.h>
  50 #include <openssl/rsa.h>
  51 #if OPENSSL_VERSION_NUMBER > 0x00908000L
  52 #include <openssl/bn.h>
  53 #endif
  54 #ifdef USE_ENGINE
  55 #include <openssl/engine.h>
  56 #endif
  57
  58 /*
  59  * Limit the size of public exponents.
  60  */
  61 #ifndef RSA_MAX_PUBEXP_BITS
  62 #define RSA_MAX_PUBEXP_BITS    35
  63 #endif
  64
  65 /*
  66  * We don't use configure for windows so enforce the OpenSSL version
  67  * here.  Unlike with configure we don't support overriding this test.
  68  */
  69 #ifdef WIN32
  70 #if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
  71        OPENSSL_VERSION_NUMBER < 0x00908000L) || \
  72       OPENSSL_VERSION_NUMBER >= 0x0090804fL)
  73 #error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
  74 #endif
  75 #endif
  76
  77
  78         /*
  79          * XXXMPA  Temporarily disable RSA_BLINDING as it requires
  80          * good quality random data that cannot currently be guaranteed.
  81          * XXXMPA  Find which versions of openssl use pseudo random data
  82          * and set RSA_FLAG_BLINDING for those.
  83          */
  84
  85 #if 0
  86 #if OPENSSL_VERSION_NUMBER < 0x0090601fL
  87 #define SET_FLAGS(rsa) \
  88         do { \
  89         (rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
  90         (rsa)->flags |= RSA_FLAG_BLINDING; \
  91         } while (0)
  92 #else
  93 #define SET_FLAGS(rsa) \
  94         do { \
  95                 (rsa)->flags |= RSA_FLAG_BLINDING; \
  96         } while (0)
  97 #endif
  98 #endif
  99
 100 #if OPENSSL_VERSION_NUMBER < 0x0090601fL
 101 #define SET_FLAGS(rsa) \
 102         do { \
 103         (rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
 104         (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 105         } while (0)
 106 #elif defined(RSA_FLAG_NO_BLINDING)
 107 #define SET_FLAGS(rsa) \
 108         do { \
 109                 (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 110                 (rsa)->flags |= RSA_FLAG_NO_BLINDING; \
 111         } while (0)
 112 #else
 113 #define SET_FLAGS(rsa) \
 114         do { \
 115                 (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 116         } while (0)
 117 #endif
 118
 119 #define DST_RET(a) {ret = a; goto err;}
 120
 121 static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);
 122
 123 static isc_result_t
 124 opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
 125 #if USE_EVP
 126         EVP_MD_CTX *evp_md_ctx;
 127         const EVP_MD *type = NULL;
 128 #endif
 129
 130         UNUSED(key);
 131         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 132                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 133                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 134                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 135                 dctx->key->key_alg == DST_ALG_RSASHA512);
 136
 137 #if USE_EVP
 138         evp_md_ctx = EVP_MD_CTX_create();
 139         if (evp_md_ctx == NULL)
 140                 return (ISC_R_NOMEMORY);
 141
 142         switch (dctx->key->key_alg) {
 143         case DST_ALG_RSAMD5:
 144                 type = EVP_md5();       /* MD5 + RSA */
 145                 break;
 146         case DST_ALG_RSASHA1:
 147         case DST_ALG_NSEC3RSASHA1:
 148                 type = EVP_sha1();      /* SHA1 + RSA */
 149                 break;
 150 #ifdef HAVE_EVP_SHA256
 151         case DST_ALG_RSASHA256:
 152                 type = EVP_sha256();    /* SHA256 + RSA */
 153                 break;
 154 #endif
 155 #ifdef HAVE_EVP_SHA512
 156         case DST_ALG_RSASHA512:
 157                 type = EVP_sha512();
 158                 break;
 159 #endif
 160         default:
 161                 INSIST(0);
 162         }
 163
 164         if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
 165                 EVP_MD_CTX_destroy(evp_md_ctx);
 166                 return (dst__openssl_toresult3(dctx->category,
 167                                                "EVP_DigestInit_ex",
 168                                                ISC_R_FAILURE));
 169         }
 170         dctx->ctxdata.evp_md_ctx = evp_md_ctx;
 171 #else
 172         switch (dctx->key->key_alg) {
 173         case DST_ALG_RSAMD5:
 174                 {
 175                         isc_md5_t *md5ctx;
 176
 177                         md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
 178                         if (md5ctx == NULL)
 179                                 return (ISC_R_NOMEMORY);
 180                         isc_md5_init(md5ctx);
 181                         dctx->ctxdata.md5ctx = md5ctx;
 182                 }
 183                 break;
 184         case DST_ALG_RSASHA1:
 185         case DST_ALG_NSEC3RSASHA1:
 186                 {
 187                         isc_sha1_t *sha1ctx;
 188
 189                         sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
 190                         if (sha1ctx == NULL)
 191                                 return (ISC_R_NOMEMORY);
 192                         isc_sha1_init(sha1ctx);
 193                         dctx->ctxdata.sha1ctx = sha1ctx;
 194                 }
 195                 break;
 196         case DST_ALG_RSASHA256:
 197                 {
 198                         isc_sha256_t *sha256ctx;
 199
 200                         sha256ctx = isc_mem_get(dctx->mctx,
 201                                                 sizeof(isc_sha256_t));
 202                         if (sha256ctx == NULL)
 203                                 return (ISC_R_NOMEMORY);
 204                         isc_sha256_init(sha256ctx);
 205                         dctx->ctxdata.sha256ctx = sha256ctx;
 206                 }
 207                 break;
 208         case DST_ALG_RSASHA512:
 209                 {
 210                         isc_sha512_t *sha512ctx;
 211
 212                         sha512ctx = isc_mem_get(dctx->mctx,
 213                                                 sizeof(isc_sha512_t));
 214                         if (sha512ctx == NULL)
 215                                 return (ISC_R_NOMEMORY);
 216                         isc_sha512_init(sha512ctx);
 217                         dctx->ctxdata.sha512ctx = sha512ctx;
 218                 }
 219                 break;
 220         default:
 221                 INSIST(0);
 222         }
 223 #endif
 224
 225         return (ISC_R_SUCCESS);
 226 }
 227
 228 static void
 229 opensslrsa_destroyctx(dst_context_t *dctx) {
 230 #if USE_EVP
 231         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 232 #endif
 233
 234         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 235                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 236                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 237                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 238                 dctx->key->key_alg == DST_ALG_RSASHA512);
 239
 240 #if USE_EVP
 241         if (evp_md_ctx != NULL) {
 242                 EVP_MD_CTX_destroy(evp_md_ctx);
 243                 dctx->ctxdata.evp_md_ctx = NULL;
 244         }
 245 #else
 246         switch (dctx->key->key_alg) {
 247         case DST_ALG_RSAMD5:
 248                 {
 249                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 250
 251                         if (md5ctx != NULL) {
 252                                 isc_md5_invalidate(md5ctx);
 253                                 isc_mem_put(dctx->mctx, md5ctx,
 254                                             sizeof(isc_md5_t));
 255                                 dctx->ctxdata.md5ctx = NULL;
 256                         }
 257                 }
 258                 break;
 259         case DST_ALG_RSASHA1:
 260         case DST_ALG_NSEC3RSASHA1:
 261                 {
 262                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 263
 264                         if (sha1ctx != NULL) {
 265                                 isc_sha1_invalidate(sha1ctx);
 266                                 isc_mem_put(dctx->mctx, sha1ctx,
 267                                             sizeof(isc_sha1_t));
 268                                 dctx->ctxdata.sha1ctx = NULL;
 269                         }
 270                 }
 271                 break;
 272         case DST_ALG_RSASHA256:
 273                 {
 274                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 275
 276                         if (sha256ctx != NULL) {
 277                                 isc_sha256_invalidate(sha256ctx);
 278                                 isc_mem_put(dctx->mctx, sha256ctx,
 279                                             sizeof(isc_sha256_t));
 280                                 dctx->ctxdata.sha256ctx = NULL;
 281                         }
 282                 }
 283                 break;
 284         case DST_ALG_RSASHA512:
 285                 {
 286                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 287
 288                         if (sha512ctx != NULL) {
 289                                 isc_sha512_invalidate(sha512ctx);
 290                                 isc_mem_put(dctx->mctx, sha512ctx,
 291                                             sizeof(isc_sha512_t));
 292                                 dctx->ctxdata.sha512ctx = NULL;
 293                         }
 294                 }
 295                 break;
 296         default:
 297                 INSIST(0);
 298         }
 299 #endif
 300 }
 301
 302 static isc_result_t
 303 opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
 304 #if USE_EVP
 305         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 306 #endif
 307
 308         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 309                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 310                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 311                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 312                 dctx->key->key_alg == DST_ALG_RSASHA512);
 313
 314 #if USE_EVP
 315         if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
 316                 return (dst__openssl_toresult3(dctx->category,
 317                                                "EVP_DigestUpdate",
 318                                                ISC_R_FAILURE));
 319         }
 320 #else
 321         switch (dctx->key->key_alg) {
 322         case DST_ALG_RSAMD5:
 323                 {
 324                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 325
 326                         isc_md5_update(md5ctx, data->base, data->length);
 327                 }
 328                 break;
 329         case DST_ALG_RSASHA1:
 330         case DST_ALG_NSEC3RSASHA1:
 331                 {
 332                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 333
 334                         isc_sha1_update(sha1ctx, data->base, data->length);
 335                 }
 336                 break;
 337         case DST_ALG_RSASHA256:
 338                 {
 339                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 340
 341                         isc_sha256_update(sha256ctx, data->base, data->length);
 342                 }
 343                 break;
 344         case DST_ALG_RSASHA512:
 345                 {
 346                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 347
 348                         isc_sha512_update(sha512ctx, data->base, data->length);
 349                 }
 350                 break;
 351         default:
 352                 INSIST(0);
 353         }
 354 #endif
 355         return (ISC_R_SUCCESS);
 356 }
 357
 358 #if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
 359 /*
 360  * Digest prefixes from RFC 5702.
 361  */
 362 static unsigned char sha256_prefix[] =
 363          { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
 364            0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
 365 static unsigned char sha512_prefix[] =
 366          { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
 367            0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
 368 #define PREFIXLEN sizeof(sha512_prefix)
 369 #else
 370 #define PREFIXLEN 0
 371 #endif
 372
 373 static isc_result_t
 374 opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 375         dst_key_t *key = dctx->key;
 376         isc_region_t r;
 377         unsigned int siglen = 0;
 378 #if USE_EVP
 379         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 380         EVP_PKEY *pkey = key->keydata.pkey;
 381 #else
 382         RSA *rsa = key->keydata.rsa;
 383         /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
 384         unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
 385         int status;
 386         int type = 0;
 387         unsigned int digestlen = 0;
 388 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 389         unsigned int prefixlen = 0;
 390         const unsigned char *prefix = NULL;
 391 #endif
 392 #endif
 393
 394         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 395                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 396                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 397                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 398                 dctx->key->key_alg == DST_ALG_RSASHA512);
 399
 400         isc_buffer_availableregion(sig, &r);
 401
 402 #if USE_EVP
 403         if (r.length < (unsigned int) EVP_PKEY_size(pkey))
 404                 return (ISC_R_NOSPACE);
 405
 406         if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
 407                 return (dst__openssl_toresult3(dctx->category,
 408                                                "EVP_SignFinal",
 409                                                ISC_R_FAILURE));
 410         }
 411 #else
 412         if (r.length < (unsigned int) RSA_size(rsa))
 413                 return (ISC_R_NOSPACE);
 414
 415         switch (dctx->key->key_alg) {
 416         case DST_ALG_RSAMD5:
 417                 {
 418                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 419
 420                         isc_md5_final(md5ctx, digest);
 421                         type = NID_md5;
 422                         digestlen = ISC_MD5_DIGESTLENGTH;
 423                 }
 424                 break;
 425         case DST_ALG_RSASHA1:
 426         case DST_ALG_NSEC3RSASHA1:
 427                 {
 428                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 429
 430                         isc_sha1_final(sha1ctx, digest);
 431                         type = NID_sha1;
 432                         digestlen = ISC_SHA1_DIGESTLENGTH;
 433                 }
 434                 break;
 435         case DST_ALG_RSASHA256:
 436                 {
 437                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 438
 439                         isc_sha256_final(digest, sha256ctx);
 440                         digestlen = ISC_SHA256_DIGESTLENGTH;
 441 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 442                         prefix = sha256_prefix;
 443                         prefixlen = sizeof(sha256_prefix);
 444 #else
 445                         type = NID_sha256;
 446 #endif
 447                 }
 448                 break;
 449         case DST_ALG_RSASHA512:
 450                 {
 451                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 452
 453                         isc_sha512_final(digest, sha512ctx);
 454                         digestlen = ISC_SHA512_DIGESTLENGTH;
 455 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 456                         prefix = sha512_prefix;
 457                         prefixlen = sizeof(sha512_prefix);
 458 #else
 459                         type = NID_sha512;
 460 #endif
 461                 }
 462                 break;
 463         default:
 464                 INSIST(0);
 465         }
 466
 467 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 468         switch (dctx->key->key_alg) {
 469         case DST_ALG_RSAMD5:
 470         case DST_ALG_RSASHA1:
 471         case DST_ALG_NSEC3RSASHA1:
 472                 INSIST(type != 0);
 473                 status = RSA_sign(type, digest, digestlen, r.base,
 474                                   &siglen, rsa);
 475                 break;
 476
 477         case DST_ALG_RSASHA256:
 478         case DST_ALG_RSASHA512:
 479                 INSIST(prefix != NULL);
 480                 INSIST(prefixlen != 0);
 481                 INSIST(prefixlen + digestlen <= sizeof(digest));
 482
 483                 memmove(digest + prefixlen, digest, digestlen);
 484                 memmove(digest, prefix, prefixlen);
 485                 status = RSA_private_encrypt(digestlen + prefixlen,
 486                                              digest, r.base, rsa,
 487                                              RSA_PKCS1_PADDING);
 488                 if (status < 0)
 489                         status = 0;
 490                 else
 491                         siglen = status;
 492                 break;
 493
 494         default:
 495                 INSIST(0);
 496         }
 497 #else
 498         INSIST(type != 0);
 499         status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
 500 #endif
 501         if (status == 0)
 502                 return (dst__openssl_toresult3(dctx->category,
 503                                                "RSA_sign",
 504                                                DST_R_OPENSSLFAILURE));
 505 #endif
 506
 507         isc_buffer_add(sig, siglen);
 508
 509         return (ISC_R_SUCCESS);
 510 }
 511
 512 static isc_result_t
 513 opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
 514         dst_key_t *key = dctx->key;
 515         int status = 0;
 516 #if USE_EVP
 517         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 518         EVP_PKEY *pkey = key->keydata.pkey;
 519         RSA *rsa;
 520         int bits;
 521 #else
 522         /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
 523         unsigned char digest[ISC_SHA512_DIGESTLENGTH];
 524         int type = 0;
 525         unsigned int digestlen = 0;
 526         RSA *rsa = key->keydata.rsa;
 527 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 528         unsigned int prefixlen = 0;
 529         const unsigned char *prefix = NULL;
 530 #endif
 531 #endif
 532
 533         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 534                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 535                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 536                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 537                 dctx->key->key_alg == DST_ALG_RSASHA512);
 538
 539 #if USE_EVP
 540         rsa = EVP_PKEY_get1_RSA(pkey);
 541         if (rsa == NULL)
 542                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 543         bits = BN_num_bits(rsa->e);
 544         RSA_free(rsa);
 545         if (bits > maxbits && maxbits != 0)
 546                 return (DST_R_VERIFYFAILURE);
 547
 548         status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
 549         switch (status) {
 550         case 1:
 551                 return (ISC_R_SUCCESS);
 552         case 0:
 553                 return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
 554         default:
 555                 return (dst__openssl_toresult3(dctx->category,
 556                                                "EVP_VerifyFinal",
 557                                                DST_R_VERIFYFAILURE));
 558         }
 559 #else
 560         if (BN_num_bits(rsa->e) > maxbits && maxbits != 0)
 561                 return (DST_R_VERIFYFAILURE);
 562
 563         switch (dctx->key->key_alg) {
 564         case DST_ALG_RSAMD5:
 565                 {
 566                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 567
 568                         isc_md5_final(md5ctx, digest);
 569                         type = NID_md5;
 570                         digestlen = ISC_MD5_DIGESTLENGTH;
 571                 }
 572                 break;
 573         case DST_ALG_RSASHA1:
 574         case DST_ALG_NSEC3RSASHA1:
 575                 {
 576                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 577
 578                         isc_sha1_final(sha1ctx, digest);
 579                         type = NID_sha1;
 580                         digestlen = ISC_SHA1_DIGESTLENGTH;
 581                 }
 582                 break;
 583         case DST_ALG_RSASHA256:
 584                 {
 585                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 586
 587                         isc_sha256_final(digest, sha256ctx);
 588                         digestlen = ISC_SHA256_DIGESTLENGTH;
 589 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 590                         prefix = sha256_prefix;
 591                         prefixlen = sizeof(sha256_prefix);
 592 #else
 593                         type = NID_sha256;
 594 #endif
 595                 }
 596                 break;
 597         case DST_ALG_RSASHA512:
 598                 {
 599                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 600
 601                         isc_sha512_final(digest, sha512ctx);
 602                         digestlen = ISC_SHA512_DIGESTLENGTH;
 603 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 604                         prefix = sha512_prefix;
 605                         prefixlen = sizeof(sha512_prefix);
 606 #else
 607                         type = NID_sha512;
 608 #endif
 609                 }
 610                 break;
 611         default:
 612                 INSIST(0);
 613         }
 614
 615         if (sig->length != (unsigned int) RSA_size(rsa))
 616                 return (DST_R_VERIFYFAILURE);
 617
 618 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 619         switch (dctx->key->key_alg) {
 620         case DST_ALG_RSAMD5:
 621         case DST_ALG_RSASHA1:
 622         case DST_ALG_NSEC3RSASHA1:
 623                 INSIST(type != 0);
 624                 status = RSA_verify(type, digest, digestlen, sig->base,
 625                                     RSA_size(rsa), rsa);
 626                 break;
 627
 628         case DST_ALG_RSASHA256:
 629         case DST_ALG_RSASHA512:
 630                 {
 631                         /*
 632                          * 1024 is big enough for all valid RSA bit sizes
 633                          * for use with DNSSEC.
 634                          */
 635                         unsigned char original[PREFIXLEN + 1024];
 636
 637                         INSIST(prefix != NULL);
 638                         INSIST(prefixlen != 0U);
 639
 640                         if (RSA_size(rsa) > (int)sizeof(original))
 641                                 return (DST_R_VERIFYFAILURE);
 642
 643                         status = RSA_public_decrypt(sig->length, sig->base,
 644                                                     original, rsa,
 645                                                     RSA_PKCS1_PADDING);
 646                         if (status <= 0)
 647                                 return (dst__openssl_toresult3(
 648                                                 dctx->category,
 649                                                 "RSA_public_decrypt",
 650                                                 DST_R_VERIFYFAILURE));
 651                         if (status != (int)(prefixlen + digestlen))
 652                                 return (DST_R_VERIFYFAILURE);
 653                         if (memcmp(original, prefix, prefixlen))
 654                                 return (DST_R_VERIFYFAILURE);
 655                         if (memcmp(original + prefixlen, digest, digestlen))
 656                                 return (DST_R_VERIFYFAILURE);
 657                         status = 1;
 658                 }
 659                 break;
 660
 661         default:
 662                 INSIST(0);
 663         }
 664 #else
 665         INSIST(type != 0);
 666         status = RSA_verify(type, digest, digestlen, sig->base,
 667                              RSA_size(rsa), rsa);
 668 #endif
 669         if (status != 1)
 670                 return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
 671         return (ISC_R_SUCCESS);
 672 #endif
 673 }
 674
 675 static isc_result_t
 676 opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 677         return (opensslrsa_verify2(dctx, 0, sig));
 678 }
 679
 680 static isc_boolean_t
 681 opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
 682         int status;
 683         RSA *rsa1 = NULL, *rsa2 = NULL;
 684 #if USE_EVP
 685         EVP_PKEY *pkey1, *pkey2;
 686 #endif
 687
 688 #if USE_EVP
 689         pkey1 = key1->keydata.pkey;
 690         pkey2 = key2->keydata.pkey;
 691         /*
 692          * The pkey reference will keep these around after
 693          * the RSA_free() call.
 694          */
 695         if (pkey1 != NULL) {
 696                 rsa1 = EVP_PKEY_get1_RSA(pkey1);
 697                 RSA_free(rsa1);
 698         }
 699         if (pkey2 != NULL) {
 700                 rsa2 = EVP_PKEY_get1_RSA(pkey2);
 701                 RSA_free(rsa2);
 702         }
 703 #else
 704         rsa1 = key1->keydata.rsa;
 705         rsa2 = key2->keydata.rsa;
 706 #endif
 707
 708         if (rsa1 == NULL && rsa2 == NULL)
 709                 return (ISC_TRUE);
 710         else if (rsa1 == NULL || rsa2 == NULL)
 711                 return (ISC_FALSE);
 712
 713         status = BN_cmp(rsa1->n, rsa2->n) ||
 714                  BN_cmp(rsa1->e, rsa2->e);
 715
 716         if (status != 0)
 717                 return (ISC_FALSE);
 718
 719 #if USE_EVP
 720         if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
 721             (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
 722                 if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
 723                     (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
 724                         return (ISC_FALSE);
 725                 /*
 726                  * Can't compare private parameters, BTW does it make sense?
 727                  */
 728                 return (ISC_TRUE);
 729         }
 730 #endif
 731
 732         if (rsa1->d != NULL || rsa2->d != NULL) {
 733                 if (rsa1->d == NULL || rsa2->d == NULL)
 734                         return (ISC_FALSE);
 735                 status = BN_cmp(rsa1->d, rsa2->d) ||
 736                          BN_cmp(rsa1->p, rsa2->p) ||
 737                          BN_cmp(rsa1->q, rsa2->q);
 738
 739                 if (status != 0)
 740                         return (ISC_FALSE);
 741         }
 742         return (ISC_TRUE);
 743 }
 744
 745 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 746 static int
 747 progress_cb(int p, int n, BN_GENCB *cb)
 748 {
 749         union {
 750                 void *dptr;
 751                 void (*fptr)(int);
 752         } u;
 753
 754         UNUSED(n);
 755
 756         u.dptr = cb->arg;
 757         if (u.fptr != NULL)
 758                 u.fptr(p);
 759         return (1);
 760 }
 761 #endif
 762
 763 static isc_result_t
 764 opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 765 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 766         isc_result_t ret = DST_R_OPENSSLFAILURE;
 767         BN_GENCB cb;
 768         union {
 769                 void *dptr;
 770                 void (*fptr)(int);
 771         } u;
 772         RSA *rsa = RSA_new();
 773         BIGNUM *e = BN_new();
 774 #if USE_EVP
 775         EVP_PKEY *pkey = EVP_PKEY_new();
 776 #endif
 777
 778         if (rsa == NULL || e == NULL)
 779                 goto err;
 780 #if USE_EVP
 781         if (pkey == NULL)
 782                 goto err;
 783         if (!EVP_PKEY_set1_RSA(pkey, rsa))
 784                 goto err;
 785 #endif
 786
 787         if (exp == 0) {
 788                 /* RSA_F4 0x10001 */
 789                 BN_set_bit(e, 0);
 790                 BN_set_bit(e, 16);
 791         } else {
 792                 /* (phased-out) F5 0x100000001 */
 793                 BN_set_bit(e, 0);
 794                 BN_set_bit(e, 32);
 795         }
 796
 797         if (callback == NULL) {
 798                 BN_GENCB_set_old(&cb, NULL, NULL);
 799         } else {
 800                 u.fptr = callback;
 801                 BN_GENCB_set(&cb, &progress_cb, u.dptr);
 802         }
 803
 804         if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
 805                 BN_free(e);
 806                 SET_FLAGS(rsa);
 807 #if USE_EVP
 808                 key->keydata.pkey = pkey;
 809
 810                 RSA_free(rsa);
 811 #else
 812                 key->keydata.rsa = rsa;
 813 #endif
 814                 return (ISC_R_SUCCESS);
 815         }
 816         ret = dst__openssl_toresult2("RSA_generate_key_ex",
 817                                      DST_R_OPENSSLFAILURE);
 818
 819 err:
 820 #if USE_EVP
 821         if (pkey != NULL)
 822                 EVP_PKEY_free(pkey);
 823 #endif
 824         if (e != NULL)
 825                 BN_free(e);
 826         if (rsa != NULL)
 827                 RSA_free(rsa);
 828         return (dst__openssl_toresult(ret));
 829 #else
 830         RSA *rsa;
 831         unsigned long e;
 832 #if USE_EVP
 833         EVP_PKEY *pkey = EVP_PKEY_new();
 834
 835         UNUSED(callback);
 836
 837         if (pkey == NULL)
 838                 return (ISC_R_NOMEMORY);
 839 #else
 840         UNUSED(callback);
 841 #endif
 842
 843         if (exp == 0)
 844                e = RSA_F4;
 845         else
 846                e = 0x40000003;
 847         rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
 848         if (rsa == NULL) {
 849 #if USE_EVP
 850                 EVP_PKEY_free(pkey);
 851 #endif
 852                 return (dst__openssl_toresult2("RSA_generate_key",
 853                                                DST_R_OPENSSLFAILURE));
 854         }
 855         SET_FLAGS(rsa);
 856 #if USE_EVP
 857         if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
 858                 EVP_PKEY_free(pkey);
 859                 RSA_free(rsa);
 860                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 861         }
 862         key->keydata.pkey = pkey;
 863         RSA_free(rsa);
 864 #else
 865         key->keydata.rsa = rsa;
 866 #endif
 867
 868         return (ISC_R_SUCCESS);
 869 #endif
 870 }
 871
 872 static isc_boolean_t
 873 opensslrsa_isprivate(const dst_key_t *key) {
 874 #if USE_EVP
 875         RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
 876         INSIST(rsa != NULL);
 877         RSA_free(rsa);
 878         /* key->keydata.pkey still has a reference so rsa is still valid. */
 879 #else
 880         RSA *rsa = key->keydata.rsa;
 881 #endif
 882         if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
 883                 return (ISC_TRUE);
 884         return (ISC_TF(rsa != NULL && rsa->d != NULL));
 885 }
 886
 887 static void
 888 opensslrsa_destroy(dst_key_t *key) {
 889 #if USE_EVP
 890         EVP_PKEY *pkey = key->keydata.pkey;
 891         EVP_PKEY_free(pkey);
 892         key->keydata.pkey = NULL;
 893 #else
 894         RSA *rsa = key->keydata.rsa;
 895         RSA_free(rsa);
 896         key->keydata.rsa = NULL;
 897 #endif
 898 }
 899
 900
 901 static isc_result_t
 902 opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
 903         isc_region_t r;
 904         unsigned int e_bytes;
 905         unsigned int mod_bytes;
 906         isc_result_t ret;
 907         RSA *rsa;
 908 #if USE_EVP
 909         EVP_PKEY *pkey;
 910 #endif
 911
 912 #if USE_EVP
 913         REQUIRE(key->keydata.pkey != NULL);
 914 #else
 915         REQUIRE(key->keydata.rsa != NULL);
 916 #endif
 917
 918 #if USE_EVP
 919         pkey = key->keydata.pkey;
 920         rsa = EVP_PKEY_get1_RSA(pkey);
 921         if (rsa == NULL)
 922                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 923 #else
 924         rsa = key->keydata.rsa;
 925 #endif
 926
 927         isc_buffer_availableregion(data, &r);
 928
 929         e_bytes = BN_num_bytes(rsa->e);
 930         mod_bytes = BN_num_bytes(rsa->n);
 931
 932         if (e_bytes < 256) {    /*%< key exponent is <= 2040 bits */
 933                 if (r.length < 1)
 934                         DST_RET(ISC_R_NOSPACE);
 935                 isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
 936                 isc_region_consume(&r, 1);
 937         } else {
 938                 if (r.length < 3)
 939                         DST_RET(ISC_R_NOSPACE);
 940                 isc_buffer_putuint8(data, 0);
 941                 isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
 942                 isc_region_consume(&r, 3);
 943         }
 944
 945         if (r.length < e_bytes + mod_bytes)
 946                 DST_RET(ISC_R_NOSPACE);
 947
 948         BN_bn2bin(rsa->e, r.base);
 949         isc_region_consume(&r, e_bytes);
 950         BN_bn2bin(rsa->n, r.base);
 951
 952         isc_buffer_add(data, e_bytes + mod_bytes);
 953
 954         ret = ISC_R_SUCCESS;
 955  err:
 956 #if USE_EVP
 957         if (rsa != NULL)
 958                 RSA_free(rsa);
 959 #endif
 960         return (ret);
 961 }
 962
 963 static isc_result_t
 964 opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 965         RSA *rsa;
 966         isc_region_t r;
 967         unsigned int e_bytes;
 968 #if USE_EVP
 969         EVP_PKEY *pkey;
 970 #endif
 971
 972         isc_buffer_remainingregion(data, &r);
 973         if (r.length == 0)
 974                 return (ISC_R_SUCCESS);
 975
 976         rsa = RSA_new();
 977         if (rsa == NULL)
 978                 return (dst__openssl_toresult(ISC_R_NOMEMORY));
 979         SET_FLAGS(rsa);
 980
 981         if (r.length < 1) {
 982                 RSA_free(rsa);
 983                 return (DST_R_INVALIDPUBLICKEY);
 984         }
 985         e_bytes = *r.base++;
 986         r.length--;
 987
 988         if (e_bytes == 0) {
 989                 if (r.length < 2) {
 990                         RSA_free(rsa);
 991                         return (DST_R_INVALIDPUBLICKEY);
 992                 }
 993                 e_bytes = ((*r.base++) << 8);
 994                 e_bytes += *r.base++;
 995                 r.length -= 2;
 996         }
 997
 998         if (r.length < e_bytes) {
 999                 RSA_free(rsa);
1000                 return (DST_R_INVALIDPUBLICKEY);
1001         }
1002         rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
1003         r.base += e_bytes;
1004         r.length -= e_bytes;
1005
1006         rsa->n = BN_bin2bn(r.base, r.length, NULL);
1007
1008         key->key_size = BN_num_bits(rsa->n);
1009
1010         isc_buffer_forward(data, r.length);
1011
1012 #if USE_EVP
1013         pkey = EVP_PKEY_new();
1014         if (pkey == NULL) {
1015                 RSA_free(rsa);
1016                 return (ISC_R_NOMEMORY);
1017         }
1018         if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
1019                 EVP_PKEY_free(pkey);
1020                 RSA_free(rsa);
1021                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1022         }
1023         key->keydata.pkey = pkey;
1024         RSA_free(rsa);
1025 #else
1026         key->keydata.rsa = rsa;
1027 #endif
1028
1029         return (ISC_R_SUCCESS);
1030 }
1031
1032 static isc_result_t
1033 opensslrsa_tofile(const dst_key_t *key, const char *directory) {
1034         int i;
1035         RSA *rsa;
1036         dst_private_t priv;
1037         unsigned char *bufs[8];
1038         isc_result_t result;
1039
1040 #if USE_EVP
1041         if (key->keydata.pkey == NULL)
1042                 return (DST_R_NULLKEY);
1043         rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
1044         if (rsa == NULL)
1045                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1046 #else
1047         if (key->keydata.rsa == NULL)
1048                 return (DST_R_NULLKEY);
1049         rsa = key->keydata.rsa;
1050 #endif
1051         memset(bufs, 0, sizeof(bufs));
1052
1053         if (key->external) {
1054                 priv.nelements = 0;
1055                 result = dst__privstruct_writefile(key, &priv, directory);
1056                 goto fail;
1057         }
1058
1059         for (i = 0; i < 8; i++) {
1060                 bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
1061                 if (bufs[i] == NULL) {
1062                         result = ISC_R_NOMEMORY;
1063                         goto fail;
1064                 }
1065         }
1066
1067         i = 0;
1068
1069         priv.elements[i].tag = TAG_RSA_MODULUS;
1070         priv.elements[i].length = BN_num_bytes(rsa->n);
1071         BN_bn2bin(rsa->n, bufs[i]);
1072         priv.elements[i].data = bufs[i];
1073         i++;
1074
1075         priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
1076         priv.elements[i].length = BN_num_bytes(rsa->e);
1077         BN_bn2bin(rsa->e, bufs[i]);
1078         priv.elements[i].data = bufs[i];
1079         i++;
1080
1081         if (rsa->d != NULL) {
1082                 priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
1083                 priv.elements[i].length = BN_num_bytes(rsa->d);
1084                 BN_bn2bin(rsa->d, bufs[i]);
1085                 priv.elements[i].data = bufs[i];
1086                 i++;
1087         }
1088
1089         if (rsa->p != NULL) {
1090                 priv.elements[i].tag = TAG_RSA_PRIME1;
1091                 priv.elements[i].length = BN_num_bytes(rsa->p);
1092                 BN_bn2bin(rsa->p, bufs[i]);
1093                 priv.elements[i].data = bufs[i];
1094                 i++;
1095         }
1096
1097         if (rsa->q != NULL) {
1098                 priv.elements[i].tag = TAG_RSA_PRIME2;
1099                 priv.elements[i].length = BN_num_bytes(rsa->q);
1100                 BN_bn2bin(rsa->q, bufs[i]);
1101                 priv.elements[i].data = bufs[i];
1102                 i++;
1103         }
1104
1105         if (rsa->dmp1 != NULL) {
1106                 priv.elements[i].tag = TAG_RSA_EXPONENT1;
1107                 priv.elements[i].length = BN_num_bytes(rsa->dmp1);
1108                 BN_bn2bin(rsa->dmp1, bufs[i]);
1109                 priv.elements[i].data = bufs[i];
1110                 i++;
1111         }
1112
1113         if (rsa->dmq1 != NULL) {
1114                 priv.elements[i].tag = TAG_RSA_EXPONENT2;
1115                 priv.elements[i].length = BN_num_bytes(rsa->dmq1);
1116                 BN_bn2bin(rsa->dmq1, bufs[i]);
1117                 priv.elements[i].data = bufs[i];
1118                 i++;
1119         }
1120
1121         if (rsa->iqmp != NULL) {
1122                 priv.elements[i].tag = TAG_RSA_COEFFICIENT;
1123                 priv.elements[i].length = BN_num_bytes(rsa->iqmp);
1124                 BN_bn2bin(rsa->iqmp, bufs[i]);
1125                 priv.elements[i].data = bufs[i];
1126                 i++;
1127         }
1128
1129         if (key->engine != NULL) {
1130                 priv.elements[i].tag = TAG_RSA_ENGINE;
1131                 priv.elements[i].length = strlen(key->engine) + 1;
1132                 priv.elements[i].data = (unsigned char *)key->engine;
1133                 i++;
1134         }
1135
1136         if (key->label != NULL) {
1137                 priv.elements[i].tag = TAG_RSA_LABEL;
1138                 priv.elements[i].length = strlen(key->label) + 1;
1139                 priv.elements[i].data = (unsigned char *)key->label;
1140                 i++;
1141         }
1142
1143
1144         priv.nelements = i;
1145         result = dst__privstruct_writefile(key, &priv, directory);
1146  fail:
1147 #if USE_EVP
1148         RSA_free(rsa);
1149 #endif
1150         for (i = 0; i < 8; i++) {
1151                 if (bufs[i] == NULL)
1152                         break;
1153                 isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
1154         }
1155         return (result);
1156 }
1157
1158 static isc_result_t
1159 rsa_check(RSA *rsa, RSA *pub)
1160 {
1161         /* Public parameters should be the same but if they are not set
1162          * copy them from the public key. */
1163         if (pub != NULL) {
1164                 if (rsa->n != NULL) {
1165                         if (BN_cmp(rsa->n, pub->n) != 0)
1166                                 return (DST_R_INVALIDPRIVATEKEY);
1167                 } else {
1168                         rsa->n = pub->n;
1169                         pub->n = NULL;
1170                 }
1171                 if (rsa->e != NULL) {
1172                         if (BN_cmp(rsa->e, pub->e) != 0)
1173                                 return (DST_R_INVALIDPRIVATEKEY);
1174                 } else {
1175                         rsa->e = pub->e;
1176                         pub->e = NULL;
1177                 }
1178         }
1179         if (rsa->n == NULL || rsa->e == NULL)
1180                 return (DST_R_INVALIDPRIVATEKEY);
1181         return (ISC_R_SUCCESS);
1182 }
1183
1184 static isc_result_t
1185 opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1186         dst_private_t priv;
1187         isc_result_t ret;
1188         int i;
1189         RSA *rsa = NULL, *pubrsa = NULL;
1190 #ifdef USE_ENGINE
1191         ENGINE *e = NULL;
1192 #endif
1193         isc_mem_t *mctx = key->mctx;
1194         const char *engine = NULL, *label = NULL;
1195 #if defined(USE_ENGINE) || USE_EVP
1196         EVP_PKEY *pkey = NULL;
1197 #endif
1198
1199 #if USE_EVP
1200         if (pub != NULL && pub->keydata.pkey != NULL)
1201                 pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
1202 #else
1203         if (pub != NULL && pub->keydata.rsa != NULL) {
1204                 pubrsa = pub->keydata.rsa;
1205                 pub->keydata.rsa = NULL;
1206         }
1207 #endif
1208
1209         /* read private key file */
1210         ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1211         if (ret != ISC_R_SUCCESS)
1212                 goto err;
1213
1214         if (key->external && priv.nelements != 0)
1215                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1216
1217         for (i = 0; i < priv.nelements; i++) {
1218                 switch (priv.elements[i].tag) {
1219                 case TAG_RSA_ENGINE:
1220                         engine = (char *)priv.elements[i].data;
1221                         break;
1222                 case TAG_RSA_LABEL:
1223                         label = (char *)priv.elements[i].data;
1224                         break;
1225                 default:
1226                         break;
1227                 }
1228         }
1229
1230         /*
1231          * Is this key is stored in a HSM?
1232          * See if we can fetch it.
1233          */
1234         if (label != NULL) {
1235 #ifdef USE_ENGINE
1236                 if (engine == NULL)
1237                         DST_RET(DST_R_NOENGINE);
1238                 e = dst__openssl_getengine(engine);
1239                 if (e == NULL)
1240                         DST_RET(DST_R_NOENGINE);
1241                 pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1242                 if (pkey == NULL)
1243                         DST_RET(dst__openssl_toresult2(
1244                                         "ENGINE_load_private_key",
1245                                         ISC_R_NOTFOUND));
1246                 key->engine = isc_mem_strdup(key->mctx, engine);
1247                 if (key->engine == NULL)
1248                         DST_RET(ISC_R_NOMEMORY);
1249                 key->label = isc_mem_strdup(key->mctx, label);
1250                 if (key->label == NULL)
1251                         DST_RET(ISC_R_NOMEMORY);
1252                 rsa = EVP_PKEY_get1_RSA(pkey);
1253                 if (rsa == NULL)
1254                         DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1255                 if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1256                         DST_RET(DST_R_INVALIDPRIVATEKEY);
1257                 if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1258                         DST_RET(ISC_R_RANGE);
1259                 if (pubrsa != NULL)
1260                         RSA_free(pubrsa);
1261                 key->key_size = EVP_PKEY_bits(pkey);
1262 #if USE_EVP
1263                 key->keydata.pkey = pkey;
1264                 RSA_free(rsa);
1265 #else
1266                 key->keydata.rsa = rsa;
1267                 EVP_PKEY_free(pkey);
1268 #endif
1269                 dst__privstruct_free(&priv, mctx);
1270                 memset(&priv, 0, sizeof(priv));
1271                 return (ISC_R_SUCCESS);
1272 #else
1273                 DST_RET(DST_R_NOENGINE);
1274 #endif
1275         }
1276
1277         rsa = RSA_new();
1278         if (rsa == NULL)
1279                 DST_RET(ISC_R_NOMEMORY);
1280         SET_FLAGS(rsa);
1281
1282 #if USE_EVP
1283         pkey = EVP_PKEY_new();
1284         if (pkey == NULL)
1285                 DST_RET(ISC_R_NOMEMORY);
1286         if (!EVP_PKEY_set1_RSA(pkey, rsa))
1287                 DST_RET(ISC_R_FAILURE);
1288         key->keydata.pkey = pkey;
1289 #else
1290         key->keydata.rsa = rsa;
1291 #endif
1292
1293         for (i = 0; i < priv.nelements; i++) {
1294                 BIGNUM *bn;
1295                 switch (priv.elements[i].tag) {
1296                 case TAG_RSA_ENGINE:
1297                         continue;
1298                 case TAG_RSA_LABEL:
1299                         continue;
1300                 case TAG_RSA_PIN:
1301                         continue;
1302                 default:
1303                         bn = BN_bin2bn(priv.elements[i].data,
1304                                        priv.elements[i].length, NULL);
1305                         if (bn == NULL)
1306                                 DST_RET(ISC_R_NOMEMORY);
1307                 }
1308
1309                 switch (priv.elements[i].tag) {
1310                         case TAG_RSA_MODULUS:
1311                                 rsa->n = bn;
1312                                 break;
1313                         case TAG_RSA_PUBLICEXPONENT:
1314                                 rsa->e = bn;
1315                                 break;
1316                         case TAG_RSA_PRIVATEEXPONENT:
1317                                 rsa->d = bn;
1318                                 break;
1319                         case TAG_RSA_PRIME1:
1320                                 rsa->p = bn;
1321                                 break;
1322                         case TAG_RSA_PRIME2:
1323                                 rsa->q = bn;
1324                                 break;
1325                         case TAG_RSA_EXPONENT1:
1326                                 rsa->dmp1 = bn;
1327                                 break;
1328                         case TAG_RSA_EXPONENT2:
1329                                 rsa->dmq1 = bn;
1330                                 break;
1331                         case TAG_RSA_COEFFICIENT:
1332                                 rsa->iqmp = bn;
1333                                 break;
1334                 }
1335         }
1336         dst__privstruct_free(&priv, mctx);
1337         memset(&priv, 0, sizeof(priv));
1338
1339         if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1340                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1341         if (!key->external) {
1342                 if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1343                         DST_RET(ISC_R_RANGE);
1344         }
1345         key->key_size = BN_num_bits(rsa->n);
1346         if (pubrsa != NULL)
1347                 RSA_free(pubrsa);
1348 #if USE_EVP
1349         RSA_free(rsa);
1350 #endif
1351
1352         return (ISC_R_SUCCESS);
1353
1354  err:
1355 #if USE_EVP
1356         if (pkey != NULL)
1357                 EVP_PKEY_free(pkey);
1358 #endif
1359         if (rsa != NULL)
1360                 RSA_free(rsa);
1361         if (pubrsa != NULL)
1362                 RSA_free(pubrsa);
1363         key->keydata.generic = NULL;
1364         dst__privstruct_free(&priv, mctx);
1365         memset(&priv, 0, sizeof(priv));
1366         return (ret);
1367 }
1368
1369 static isc_result_t
1370 opensslrsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
1371                      const char *pin)
1372 {
1373 #ifdef USE_ENGINE
1374         ENGINE *e = NULL;
1375         isc_result_t ret;
1376         EVP_PKEY *pkey = NULL;
1377         RSA *rsa = NULL, *pubrsa = NULL;
1378         char *colon;
1379
1380         UNUSED(pin);
1381
1382         if (engine == NULL)
1383                 DST_RET(DST_R_NOENGINE);
1384         e = dst__openssl_getengine(engine);
1385         if (e == NULL)
1386                 DST_RET(DST_R_NOENGINE);
1387         pkey = ENGINE_load_public_key(e, label, NULL, NULL);
1388         if (pkey != NULL) {
1389                 pubrsa = EVP_PKEY_get1_RSA(pkey);
1390                 EVP_PKEY_free(pkey);
1391                 if (pubrsa == NULL)
1392                         DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1393         }
1394         pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1395         if (pkey == NULL)
1396                 DST_RET(dst__openssl_toresult2("ENGINE_load_private_key",
1397                                                ISC_R_NOTFOUND));
1398         if (engine != NULL) {
1399                 key->engine = isc_mem_strdup(key->mctx, engine);
1400                 if (key->engine == NULL)
1401                         DST_RET(ISC_R_NOMEMORY);
1402         } else {
1403                 key->engine = isc_mem_strdup(key->mctx, label);
1404                 if (key->engine == NULL)
1405                         DST_RET(ISC_R_NOMEMORY);
1406                 colon = strchr(key->engine, ':');
1407                 if (colon != NULL)
1408                         *colon = '\0';
1409         }
1410         key->label = isc_mem_strdup(key->mctx, label);
1411         if (key->label == NULL)
1412                 DST_RET(ISC_R_NOMEMORY);
1413         rsa = EVP_PKEY_get1_RSA(pkey);
1414         if (rsa == NULL)
1415                 DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1416         if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1417                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1418         if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
1419                 DST_RET(ISC_R_RANGE);
1420         if (pubrsa != NULL)
1421                 RSA_free(pubrsa);
1422         key->key_size = EVP_PKEY_bits(pkey);
1423 #if USE_EVP
1424         key->keydata.pkey = pkey;
1425         RSA_free(rsa);
1426 #else
1427         key->keydata.rsa = rsa;
1428         EVP_PKEY_free(pkey);
1429 #endif
1430         return (ISC_R_SUCCESS);
1431
1432  err:
1433         if (rsa != NULL)
1434                 RSA_free(rsa);
1435         if (pubrsa != NULL)
1436                 RSA_free(pubrsa);
1437         if (pkey != NULL)
1438                 EVP_PKEY_free(pkey);
1439         return (ret);
1440 #else
1441         UNUSED(key);
1442         UNUSED(engine);
1443         UNUSED(label);
1444         UNUSED(pin);
1445         return(DST_R_NOENGINE);
1446 #endif
1447 }
1448
1449 static dst_func_t opensslrsa_functions = {
1450         opensslrsa_createctx,
1451         opensslrsa_destroyctx,
1452         opensslrsa_adddata,
1453         opensslrsa_sign,
1454         opensslrsa_verify,
1455         opensslrsa_verify2,
1456         NULL, /*%< computesecret */
1457         opensslrsa_compare,
1458         NULL, /*%< paramcompare */
1459         opensslrsa_generate,
1460         opensslrsa_isprivate,
1461         opensslrsa_destroy,
1462         opensslrsa_todns,
1463         opensslrsa_fromdns,
1464         opensslrsa_tofile,
1465         opensslrsa_parse,
1466         NULL, /*%< cleanup */
1467         opensslrsa_fromlabel,
1468         NULL, /*%< dump */
1469         NULL, /*%< restore */
1470 };
1471
1472 isc_result_t
1473 dst__opensslrsa_init(dst_func_t **funcp, unsigned char algorithm) {
1474         REQUIRE(funcp != NULL);
1475
1476         if (*funcp == NULL) {
1477                 switch (algorithm) {
1478                 case DST_ALG_RSASHA256:
1479 #if defined(HAVE_EVP_SHA256) || !USE_EVP
1480                         *funcp = &opensslrsa_functions;
1481 #endif
1482                         break;
1483                 case DST_ALG_RSASHA512:
1484 #if defined(HAVE_EVP_SHA512) || !USE_EVP
1485                         *funcp = &opensslrsa_functions;
1486 #endif
1487                         break;
1488                 default:
1489                         *funcp = &opensslrsa_functions;
1490                         break;
1491                 }
1492         }
1493         return (ISC_R_SUCCESS);
1494 }
1495
1496 #else /* OPENSSL */
1497
1498 #include <isc/util.h>
1499
1500 EMPTY_TRANSLATION_UNIT
1501
1502 #endif /* OPENSSL */
1503 /*! \file */