contrib/bind9/lib/dns/opensslrsa_link.c

   1 /*
   2  * Copyright (C) 2004-2009, 2011, 2012, 2014  Internet Systems Consortium, Inc. ("ISC")
   3  * Copyright (C) 2000-2003  Internet Software Consortium.
   4  *
   5  * Permission to use, copy, modify, and/or distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
  10  * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
  11  * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
  12  * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
  13  * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
  14  * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  15  * PERFORMANCE OF THIS SOFTWARE.
  16  */
  17
  18 /*
  19  * Principal Author: Brian Wellington
  20  * $Id$
  21  */
  22 #ifdef OPENSSL
  23 #include <config.h>
  24
  25 #ifndef USE_EVP
  26 #if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
  27 #define USE_EVP 0
  28 #else
  29 #define USE_EVP 1
  30 #endif
  31 #endif
  32
  33
  34 #include <isc/entropy.h>
  35 #include <isc/md5.h>
  36 #include <isc/sha1.h>
  37 #include <isc/sha2.h>
  38 #include <isc/mem.h>
  39 #include <isc/string.h>
  40 #include <isc/util.h>
  41
  42 #include <dst/result.h>
  43
  44 #include "dst_internal.h"
  45 #include "dst_openssl.h"
  46 #include "dst_parse.h"
  47
  48 #include <openssl/err.h>
  49 #include <openssl/objects.h>
  50 #include <openssl/rsa.h>
  51 #if OPENSSL_VERSION_NUMBER > 0x00908000L
  52 #include <openssl/bn.h>
  53 #endif
  54 #ifdef USE_ENGINE
  55 #include <openssl/engine.h>
  56 #endif
  57
  58 /*
  59  * We don't use configure for windows so enforce the OpenSSL version
  60  * here.  Unlike with configure we don't support overriding this test.
  61  */
  62 #ifdef WIN32
  63 #if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
  64        OPENSSL_VERSION_NUMBER < 0x00908000L) || \
  65       OPENSSL_VERSION_NUMBER >= 0x0090804fL)
  66 #error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
  67 #endif
  68 #endif
  69
  70
  71         /*
  72          * XXXMPA  Temporarily disable RSA_BLINDING as it requires
  73          * good quality random data that cannot currently be guaranteed.
  74          * XXXMPA  Find which versions of openssl use pseudo random data
  75          * and set RSA_FLAG_BLINDING for those.
  76          */
  77
  78 #if 0
  79 #if OPENSSL_VERSION_NUMBER < 0x0090601fL
  80 #define SET_FLAGS(rsa) \
  81         do { \
  82         (rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
  83         (rsa)->flags |= RSA_FLAG_BLINDING; \
  84         } while (0)
  85 #else
  86 #define SET_FLAGS(rsa) \
  87         do { \
  88                 (rsa)->flags |= RSA_FLAG_BLINDING; \
  89         } while (0)
  90 #endif
  91 #endif
  92
  93 #if OPENSSL_VERSION_NUMBER < 0x0090601fL
  94 #define SET_FLAGS(rsa) \
  95         do { \
  96         (rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
  97         (rsa)->flags &= ~RSA_FLAG_BLINDING; \
  98         } while (0)
  99 #elif defined(RSA_FLAG_NO_BLINDING)
 100 #define SET_FLAGS(rsa) \
 101         do { \
 102                 (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 103                 (rsa)->flags |= RSA_FLAG_NO_BLINDING; \
 104         } while (0)
 105 #else
 106 #define SET_FLAGS(rsa) \
 107         do { \
 108                 (rsa)->flags &= ~RSA_FLAG_BLINDING; \
 109         } while (0)
 110 #endif
 111
 112 #define DST_RET(a) {ret = a; goto err;}
 113
 114 static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);
 115
 116 static isc_result_t
 117 opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
 118 #if USE_EVP
 119         EVP_MD_CTX *evp_md_ctx;
 120         const EVP_MD *type = NULL;
 121 #endif
 122
 123         UNUSED(key);
 124         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 125                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 126                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 127                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 128                 dctx->key->key_alg == DST_ALG_RSASHA512);
 129
 130 #if USE_EVP
 131         evp_md_ctx = EVP_MD_CTX_create();
 132         if (evp_md_ctx == NULL)
 133                 return (ISC_R_NOMEMORY);
 134
 135         switch (dctx->key->key_alg) {
 136         case DST_ALG_RSAMD5:
 137                 type = EVP_md5();       /* MD5 + RSA */
 138                 break;
 139         case DST_ALG_RSASHA1:
 140         case DST_ALG_NSEC3RSASHA1:
 141                 type = EVP_sha1();      /* SHA1 + RSA */
 142                 break;
 143 #ifdef HAVE_EVP_SHA256
 144         case DST_ALG_RSASHA256:
 145                 type = EVP_sha256();    /* SHA256 + RSA */
 146                 break;
 147 #endif
 148 #ifdef HAVE_EVP_SHA512
 149         case DST_ALG_RSASHA512:
 150                 type = EVP_sha512();
 151                 break;
 152 #endif
 153         default:
 154                 INSIST(0);
 155         }
 156
 157         if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
 158                 EVP_MD_CTX_destroy(evp_md_ctx);
 159                 return (dst__openssl_toresult3(dctx->category,
 160                                                "EVP_DigestInit_ex",
 161                                                ISC_R_FAILURE));
 162         }
 163         dctx->ctxdata.evp_md_ctx = evp_md_ctx;
 164 #else
 165         switch (dctx->key->key_alg) {
 166         case DST_ALG_RSAMD5:
 167                 {
 168                         isc_md5_t *md5ctx;
 169
 170                         md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
 171                         if (md5ctx == NULL)
 172                                 return (ISC_R_NOMEMORY);
 173                         isc_md5_init(md5ctx);
 174                         dctx->ctxdata.md5ctx = md5ctx;
 175                 }
 176                 break;
 177         case DST_ALG_RSASHA1:
 178         case DST_ALG_NSEC3RSASHA1:
 179                 {
 180                         isc_sha1_t *sha1ctx;
 181
 182                         sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
 183                         if (sha1ctx == NULL)
 184                                 return (ISC_R_NOMEMORY);
 185                         isc_sha1_init(sha1ctx);
 186                         dctx->ctxdata.sha1ctx = sha1ctx;
 187                 }
 188                 break;
 189         case DST_ALG_RSASHA256:
 190                 {
 191                         isc_sha256_t *sha256ctx;
 192
 193                         sha256ctx = isc_mem_get(dctx->mctx,
 194                                                 sizeof(isc_sha256_t));
 195                         if (sha256ctx == NULL)
 196                                 return (ISC_R_NOMEMORY);
 197                         isc_sha256_init(sha256ctx);
 198                         dctx->ctxdata.sha256ctx = sha256ctx;
 199                 }
 200                 break;
 201         case DST_ALG_RSASHA512:
 202                 {
 203                         isc_sha512_t *sha512ctx;
 204
 205                         sha512ctx = isc_mem_get(dctx->mctx,
 206                                                 sizeof(isc_sha512_t));
 207                         if (sha512ctx == NULL)
 208                                 return (ISC_R_NOMEMORY);
 209                         isc_sha512_init(sha512ctx);
 210                         dctx->ctxdata.sha512ctx = sha512ctx;
 211                 }
 212                 break;
 213         default:
 214                 INSIST(0);
 215         }
 216 #endif
 217
 218         return (ISC_R_SUCCESS);
 219 }
 220
 221 static void
 222 opensslrsa_destroyctx(dst_context_t *dctx) {
 223 #if USE_EVP
 224         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 225 #endif
 226
 227         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 228                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 229                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 230                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 231                 dctx->key->key_alg == DST_ALG_RSASHA512);
 232
 233 #if USE_EVP
 234         if (evp_md_ctx != NULL) {
 235                 EVP_MD_CTX_destroy(evp_md_ctx);
 236                 dctx->ctxdata.evp_md_ctx = NULL;
 237         }
 238 #else
 239         switch (dctx->key->key_alg) {
 240         case DST_ALG_RSAMD5:
 241                 {
 242                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 243
 244                         if (md5ctx != NULL) {
 245                                 isc_md5_invalidate(md5ctx);
 246                                 isc_mem_put(dctx->mctx, md5ctx,
 247                                             sizeof(isc_md5_t));
 248                                 dctx->ctxdata.md5ctx = NULL;
 249                         }
 250                 }
 251                 break;
 252         case DST_ALG_RSASHA1:
 253         case DST_ALG_NSEC3RSASHA1:
 254                 {
 255                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 256
 257                         if (sha1ctx != NULL) {
 258                                 isc_sha1_invalidate(sha1ctx);
 259                                 isc_mem_put(dctx->mctx, sha1ctx,
 260                                             sizeof(isc_sha1_t));
 261                                 dctx->ctxdata.sha1ctx = NULL;
 262                         }
 263                 }
 264                 break;
 265         case DST_ALG_RSASHA256:
 266                 {
 267                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 268
 269                         if (sha256ctx != NULL) {
 270                                 isc_sha256_invalidate(sha256ctx);
 271                                 isc_mem_put(dctx->mctx, sha256ctx,
 272                                             sizeof(isc_sha256_t));
 273                                 dctx->ctxdata.sha256ctx = NULL;
 274                         }
 275                 }
 276                 break;
 277         case DST_ALG_RSASHA512:
 278                 {
 279                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 280
 281                         if (sha512ctx != NULL) {
 282                                 isc_sha512_invalidate(sha512ctx);
 283                                 isc_mem_put(dctx->mctx, sha512ctx,
 284                                             sizeof(isc_sha512_t));
 285                                 dctx->ctxdata.sha512ctx = NULL;
 286                         }
 287                 }
 288                 break;
 289         default:
 290                 INSIST(0);
 291         }
 292 #endif
 293 }
 294
 295 static isc_result_t
 296 opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
 297 #if USE_EVP
 298         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 299 #endif
 300
 301         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 302                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 303                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 304                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 305                 dctx->key->key_alg == DST_ALG_RSASHA512);
 306
 307 #if USE_EVP
 308         if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
 309                 return (dst__openssl_toresult3(dctx->category,
 310                                                "EVP_DigestUpdate",
 311                                                ISC_R_FAILURE));
 312         }
 313 #else
 314         switch (dctx->key->key_alg) {
 315         case DST_ALG_RSAMD5:
 316                 {
 317                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 318
 319                         isc_md5_update(md5ctx, data->base, data->length);
 320                 }
 321                 break;
 322         case DST_ALG_RSASHA1:
 323         case DST_ALG_NSEC3RSASHA1:
 324                 {
 325                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 326
 327                         isc_sha1_update(sha1ctx, data->base, data->length);
 328                 }
 329                 break;
 330         case DST_ALG_RSASHA256:
 331                 {
 332                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 333
 334                         isc_sha256_update(sha256ctx, data->base, data->length);
 335                 }
 336                 break;
 337         case DST_ALG_RSASHA512:
 338                 {
 339                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 340
 341                         isc_sha512_update(sha512ctx, data->base, data->length);
 342                 }
 343                 break;
 344         default:
 345                 INSIST(0);
 346         }
 347 #endif
 348         return (ISC_R_SUCCESS);
 349 }
 350
 351 #if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
 352 /*
 353  * Digest prefixes from RFC 5702.
 354  */
 355 static unsigned char sha256_prefix[] =
 356          { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
 357            0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
 358 static unsigned char sha512_prefix[] =
 359          { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
 360            0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
 361 #define PREFIXLEN sizeof(sha512_prefix)
 362 #else
 363 #define PREFIXLEN 0
 364 #endif
 365
 366 static isc_result_t
 367 opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 368         dst_key_t *key = dctx->key;
 369         isc_region_t r;
 370         unsigned int siglen = 0;
 371 #if USE_EVP
 372         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 373         EVP_PKEY *pkey = key->keydata.pkey;
 374 #else
 375         RSA *rsa = key->keydata.rsa;
 376         /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
 377         unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
 378         int status;
 379         int type = 0;
 380         unsigned int digestlen = 0;
 381 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 382         unsigned int prefixlen = 0;
 383         const unsigned char *prefix = NULL;
 384 #endif
 385 #endif
 386
 387         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 388                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 389                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 390                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 391                 dctx->key->key_alg == DST_ALG_RSASHA512);
 392
 393         isc_buffer_availableregion(sig, &r);
 394
 395 #if USE_EVP
 396         if (r.length < (unsigned int) EVP_PKEY_size(pkey))
 397                 return (ISC_R_NOSPACE);
 398
 399         if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
 400                 return (dst__openssl_toresult3(dctx->category,
 401                                                "EVP_SignFinal",
 402                                                ISC_R_FAILURE));
 403         }
 404 #else
 405         if (r.length < (unsigned int) RSA_size(rsa))
 406                 return (ISC_R_NOSPACE);
 407
 408         switch (dctx->key->key_alg) {
 409         case DST_ALG_RSAMD5:
 410                 {
 411                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 412
 413                         isc_md5_final(md5ctx, digest);
 414                         type = NID_md5;
 415                         digestlen = ISC_MD5_DIGESTLENGTH;
 416                 }
 417                 break;
 418         case DST_ALG_RSASHA1:
 419         case DST_ALG_NSEC3RSASHA1:
 420                 {
 421                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 422
 423                         isc_sha1_final(sha1ctx, digest);
 424                         type = NID_sha1;
 425                         digestlen = ISC_SHA1_DIGESTLENGTH;
 426                 }
 427                 break;
 428         case DST_ALG_RSASHA256:
 429                 {
 430                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 431
 432                         isc_sha256_final(digest, sha256ctx);
 433                         digestlen = ISC_SHA256_DIGESTLENGTH;
 434 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 435                         prefix = sha256_prefix;
 436                         prefixlen = sizeof(sha256_prefix);
 437 #else
 438                         type = NID_sha256;
 439 #endif
 440                 }
 441                 break;
 442         case DST_ALG_RSASHA512:
 443                 {
 444                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 445
 446                         isc_sha512_final(digest, sha512ctx);
 447                         digestlen = ISC_SHA512_DIGESTLENGTH;
 448 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 449                         prefix = sha512_prefix;
 450                         prefixlen = sizeof(sha512_prefix);
 451 #else
 452                         type = NID_sha512;
 453 #endif
 454                 }
 455                 break;
 456         default:
 457                 INSIST(0);
 458         }
 459
 460 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 461         switch (dctx->key->key_alg) {
 462         case DST_ALG_RSAMD5:
 463         case DST_ALG_RSASHA1:
 464         case DST_ALG_NSEC3RSASHA1:
 465                 INSIST(type != 0);
 466                 status = RSA_sign(type, digest, digestlen, r.base,
 467                                   &siglen, rsa);
 468                 break;
 469
 470         case DST_ALG_RSASHA256:
 471         case DST_ALG_RSASHA512:
 472                 INSIST(prefix != NULL);
 473                 INSIST(prefixlen != 0);
 474                 INSIST(prefixlen + digestlen <= sizeof(digest));
 475
 476                 memmove(digest + prefixlen, digest, digestlen);
 477                 memmove(digest, prefix, prefixlen);
 478                 status = RSA_private_encrypt(digestlen + prefixlen,
 479                                              digest, r.base, rsa,
 480                                              RSA_PKCS1_PADDING);
 481                 if (status < 0)
 482                         status = 0;
 483                 else
 484                         siglen = status;
 485                 break;
 486
 487         default:
 488                 INSIST(0);
 489         }
 490 #else
 491         INSIST(type != 0);
 492         status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
 493 #endif
 494         if (status == 0)
 495                 return (dst__openssl_toresult3(dctx->category,
 496                                                "RSA_sign",
 497                                                DST_R_OPENSSLFAILURE));
 498 #endif
 499
 500         isc_buffer_add(sig, siglen);
 501
 502         return (ISC_R_SUCCESS);
 503 }
 504
 505 static isc_result_t
 506 opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 507         dst_key_t *key = dctx->key;
 508         int status = 0;
 509 #if USE_EVP
 510         EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 511         EVP_PKEY *pkey = key->keydata.pkey;
 512 #else
 513         /* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
 514         unsigned char digest[ISC_SHA512_DIGESTLENGTH];
 515         int type = 0;
 516         unsigned int digestlen = 0;
 517         RSA *rsa = key->keydata.rsa;
 518 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 519         unsigned int prefixlen = 0;
 520         const unsigned char *prefix = NULL;
 521 #endif
 522 #endif
 523
 524         REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
 525                 dctx->key->key_alg == DST_ALG_RSASHA1 ||
 526                 dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
 527                 dctx->key->key_alg == DST_ALG_RSASHA256 ||
 528                 dctx->key->key_alg == DST_ALG_RSASHA512);
 529
 530 #if USE_EVP
 531         status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
 532         switch (status) {
 533         case 1:
 534                 return (ISC_R_SUCCESS);
 535         case 0:
 536                 return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
 537         default:
 538                 return (dst__openssl_toresult3(dctx->category,
 539                                                "EVP_VerifyFinal",
 540                                                DST_R_VERIFYFAILURE));
 541         }
 542 #else
 543         switch (dctx->key->key_alg) {
 544         case DST_ALG_RSAMD5:
 545                 {
 546                         isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;
 547
 548                         isc_md5_final(md5ctx, digest);
 549                         type = NID_md5;
 550                         digestlen = ISC_MD5_DIGESTLENGTH;
 551                 }
 552                 break;
 553         case DST_ALG_RSASHA1:
 554         case DST_ALG_NSEC3RSASHA1:
 555                 {
 556                         isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;
 557
 558                         isc_sha1_final(sha1ctx, digest);
 559                         type = NID_sha1;
 560                         digestlen = ISC_SHA1_DIGESTLENGTH;
 561                 }
 562                 break;
 563         case DST_ALG_RSASHA256:
 564                 {
 565                         isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;
 566
 567                         isc_sha256_final(digest, sha256ctx);
 568                         digestlen = ISC_SHA256_DIGESTLENGTH;
 569 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 570                         prefix = sha256_prefix;
 571                         prefixlen = sizeof(sha256_prefix);
 572 #else
 573                         type = NID_sha256;
 574 #endif
 575                 }
 576                 break;
 577         case DST_ALG_RSASHA512:
 578                 {
 579                         isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;
 580
 581                         isc_sha512_final(digest, sha512ctx);
 582                         digestlen = ISC_SHA512_DIGESTLENGTH;
 583 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 584                         prefix = sha512_prefix;
 585                         prefixlen = sizeof(sha512_prefix);
 586 #else
 587                         type = NID_sha512;
 588 #endif
 589                 }
 590                 break;
 591         default:
 592                 INSIST(0);
 593         }
 594
 595         if (sig->length != (unsigned int) RSA_size(rsa))
 596                 return (DST_R_VERIFYFAILURE);
 597
 598 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 599         switch (dctx->key->key_alg) {
 600         case DST_ALG_RSAMD5:
 601         case DST_ALG_RSASHA1:
 602         case DST_ALG_NSEC3RSASHA1:
 603                 INSIST(type != 0);
 604                 status = RSA_verify(type, digest, digestlen, sig->base,
 605                                     RSA_size(rsa), rsa);
 606                 break;
 607
 608         case DST_ALG_RSASHA256:
 609         case DST_ALG_RSASHA512:
 610                 {
 611                         /*
 612                          * 1024 is big enough for all valid RSA bit sizes
 613                          * for use with DNSSEC.
 614                          */
 615                         unsigned char original[PREFIXLEN + 1024];
 616
 617                         INSIST(prefix != NULL);
 618                         INSIST(prefixlen != 0U);
 619
 620                         if (RSA_size(rsa) > (int)sizeof(original))
 621                                 return (DST_R_VERIFYFAILURE);
 622
 623                         status = RSA_public_decrypt(sig->length, sig->base,
 624                                                     original, rsa,
 625                                                     RSA_PKCS1_PADDING);
 626                         if (status <= 0)
 627                                 return (dst__openssl_toresult3(
 628                                                 dctx->category,
 629                                                 "RSA_public_decrypt",
 630                                                 DST_R_VERIFYFAILURE));
 631                         if (status != (int)(prefixlen + digestlen))
 632                                 return (DST_R_VERIFYFAILURE);
 633                         if (memcmp(original, prefix, prefixlen))
 634                                 return (DST_R_VERIFYFAILURE);
 635                         if (memcmp(original + prefixlen, digest, digestlen))
 636                                 return (DST_R_VERIFYFAILURE);
 637                         status = 1;
 638                 }
 639                 break;
 640
 641         default:
 642                 INSIST(0);
 643         }
 644 #else
 645         INSIST(type != 0);
 646         status = RSA_verify(type, digest, digestlen, sig->base,
 647                              RSA_size(rsa), rsa);
 648 #endif
 649         if (status != 1)
 650                 return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
 651         return (ISC_R_SUCCESS);
 652 #endif
 653 }
 654
 655 static isc_boolean_t
 656 opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
 657         int status;
 658         RSA *rsa1 = NULL, *rsa2 = NULL;
 659 #if USE_EVP
 660         EVP_PKEY *pkey1, *pkey2;
 661 #endif
 662
 663 #if USE_EVP
 664         pkey1 = key1->keydata.pkey;
 665         pkey2 = key2->keydata.pkey;
 666         /*
 667          * The pkey reference will keep these around after
 668          * the RSA_free() call.
 669          */
 670         if (pkey1 != NULL) {
 671                 rsa1 = EVP_PKEY_get1_RSA(pkey1);
 672                 RSA_free(rsa1);
 673         }
 674         if (pkey2 != NULL) {
 675                 rsa2 = EVP_PKEY_get1_RSA(pkey2);
 676                 RSA_free(rsa2);
 677         }
 678 #else
 679         rsa1 = key1->keydata.rsa;
 680         rsa2 = key2->keydata.rsa;
 681 #endif
 682
 683         if (rsa1 == NULL && rsa2 == NULL)
 684                 return (ISC_TRUE);
 685         else if (rsa1 == NULL || rsa2 == NULL)
 686                 return (ISC_FALSE);
 687
 688         status = BN_cmp(rsa1->n, rsa2->n) ||
 689                  BN_cmp(rsa1->e, rsa2->e);
 690
 691         if (status != 0)
 692                 return (ISC_FALSE);
 693
 694 #if USE_EVP
 695         if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
 696             (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
 697                 if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
 698                     (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
 699                         return (ISC_FALSE);
 700                 /*
 701                  * Can't compare private parameters, BTW does it make sense?
 702                  */
 703                 return (ISC_TRUE);
 704         }
 705 #endif
 706
 707         if (rsa1->d != NULL || rsa2->d != NULL) {
 708                 if (rsa1->d == NULL || rsa2->d == NULL)
 709                         return (ISC_FALSE);
 710                 status = BN_cmp(rsa1->d, rsa2->d) ||
 711                          BN_cmp(rsa1->p, rsa2->p) ||
 712                          BN_cmp(rsa1->q, rsa2->q);
 713
 714                 if (status != 0)
 715                         return (ISC_FALSE);
 716         }
 717         return (ISC_TRUE);
 718 }
 719
 720 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 721 static int
 722 progress_cb(int p, int n, BN_GENCB *cb)
 723 {
 724         union {
 725                 void *dptr;
 726                 void (*fptr)(int);
 727         } u;
 728
 729         UNUSED(n);
 730
 731         u.dptr = cb->arg;
 732         if (u.fptr != NULL)
 733                 u.fptr(p);
 734         return (1);
 735 }
 736 #endif
 737
 738 static isc_result_t
 739 opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 740 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 741         isc_result_t ret = DST_R_OPENSSLFAILURE;
 742         BN_GENCB cb;
 743         union {
 744                 void *dptr;
 745                 void (*fptr)(int);
 746         } u;
 747         RSA *rsa = RSA_new();
 748         BIGNUM *e = BN_new();
 749 #if USE_EVP
 750         EVP_PKEY *pkey = EVP_PKEY_new();
 751 #endif
 752
 753         if (rsa == NULL || e == NULL)
 754                 goto err;
 755 #if USE_EVP
 756         if (pkey == NULL)
 757                 goto err;
 758         if (!EVP_PKEY_set1_RSA(pkey, rsa))
 759                 goto err;
 760 #endif
 761
 762         if (exp == 0) {
 763                 /* RSA_F4 0x10001 */
 764                 BN_set_bit(e, 0);
 765                 BN_set_bit(e, 16);
 766         } else {
 767                 /* F5 0x100000001 */
 768                 BN_set_bit(e, 0);
 769                 BN_set_bit(e, 32);
 770         }
 771
 772         if (callback == NULL) {
 773                 BN_GENCB_set_old(&cb, NULL, NULL);
 774         } else {
 775                 u.fptr = callback;
 776                 BN_GENCB_set(&cb, &progress_cb, u.dptr);
 777         }
 778
 779         if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
 780                 BN_free(e);
 781                 SET_FLAGS(rsa);
 782 #if USE_EVP
 783                 key->keydata.pkey = pkey;
 784
 785                 RSA_free(rsa);
 786 #else
 787                 key->keydata.rsa = rsa;
 788 #endif
 789                 return (ISC_R_SUCCESS);
 790         }
 791         ret = dst__openssl_toresult2("RSA_generate_key_ex",
 792                                      DST_R_OPENSSLFAILURE);
 793
 794 err:
 795 #if USE_EVP
 796         if (pkey != NULL)
 797                 EVP_PKEY_free(pkey);
 798 #endif
 799         if (e != NULL)
 800                 BN_free(e);
 801         if (rsa != NULL)
 802                 RSA_free(rsa);
 803         return (dst__openssl_toresult(ret));
 804 #else
 805         RSA *rsa;
 806         unsigned long e;
 807 #if USE_EVP
 808         EVP_PKEY *pkey = EVP_PKEY_new();
 809
 810         UNUSED(callback);
 811
 812         if (pkey == NULL)
 813                 return (ISC_R_NOMEMORY);
 814 #else
 815         UNUSED(callback);
 816 #endif
 817
 818         if (exp == 0)
 819                e = RSA_F4;
 820         else
 821                e = 0x40000003;
 822         rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
 823         if (rsa == NULL) {
 824 #if USE_EVP
 825                 EVP_PKEY_free(pkey);
 826 #endif
 827                 return (dst__openssl_toresult2("RSA_generate_key",
 828                                                DST_R_OPENSSLFAILURE));
 829         }
 830         SET_FLAGS(rsa);
 831 #if USE_EVP
 832         if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
 833                 EVP_PKEY_free(pkey);
 834                 RSA_free(rsa);
 835                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 836         }
 837         key->keydata.pkey = pkey;
 838         RSA_free(rsa);
 839 #else
 840         key->keydata.rsa = rsa;
 841 #endif
 842
 843         return (ISC_R_SUCCESS);
 844 #endif
 845 }
 846
 847 static isc_boolean_t
 848 opensslrsa_isprivate(const dst_key_t *key) {
 849 #if USE_EVP
 850         RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
 851         INSIST(rsa != NULL);
 852         RSA_free(rsa);
 853         /* key->keydata.pkey still has a reference so rsa is still valid. */
 854 #else
 855         RSA *rsa = key->keydata.rsa;
 856 #endif
 857         if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
 858                 return (ISC_TRUE);
 859         return (ISC_TF(rsa != NULL && rsa->d != NULL));
 860 }
 861
 862 static void
 863 opensslrsa_destroy(dst_key_t *key) {
 864 #if USE_EVP
 865         EVP_PKEY *pkey = key->keydata.pkey;
 866         EVP_PKEY_free(pkey);
 867         key->keydata.pkey = NULL;
 868 #else
 869         RSA *rsa = key->keydata.rsa;
 870         RSA_free(rsa);
 871         key->keydata.rsa = NULL;
 872 #endif
 873 }
 874
 875
 876 static isc_result_t
 877 opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
 878         isc_region_t r;
 879         unsigned int e_bytes;
 880         unsigned int mod_bytes;
 881         isc_result_t ret;
 882         RSA *rsa;
 883 #if USE_EVP
 884         EVP_PKEY *pkey;
 885 #endif
 886
 887 #if USE_EVP
 888         REQUIRE(key->keydata.pkey != NULL);
 889 #else
 890         REQUIRE(key->keydata.rsa != NULL);
 891 #endif
 892
 893 #if USE_EVP
 894         pkey = key->keydata.pkey;
 895         rsa = EVP_PKEY_get1_RSA(pkey);
 896         if (rsa == NULL)
 897                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 898 #else
 899         rsa = key->keydata.rsa;
 900 #endif
 901
 902         isc_buffer_availableregion(data, &r);
 903
 904         e_bytes = BN_num_bytes(rsa->e);
 905         mod_bytes = BN_num_bytes(rsa->n);
 906
 907         if (e_bytes < 256) {    /*%< key exponent is <= 2040 bits */
 908                 if (r.length < 1)
 909                         DST_RET(ISC_R_NOSPACE);
 910                 isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
 911                 isc_region_consume(&r, 1);
 912         } else {
 913                 if (r.length < 3)
 914                         DST_RET(ISC_R_NOSPACE);
 915                 isc_buffer_putuint8(data, 0);
 916                 isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
 917                 isc_region_consume(&r, 3);
 918         }
 919
 920         if (r.length < e_bytes + mod_bytes)
 921                 DST_RET(ISC_R_NOSPACE);
 922
 923         BN_bn2bin(rsa->e, r.base);
 924         isc_region_consume(&r, e_bytes);
 925         BN_bn2bin(rsa->n, r.base);
 926
 927         isc_buffer_add(data, e_bytes + mod_bytes);
 928
 929         ret = ISC_R_SUCCESS;
 930  err:
 931 #if USE_EVP
 932         if (rsa != NULL)
 933                 RSA_free(rsa);
 934 #endif
 935         return (ret);
 936 }
 937
 938 static isc_result_t
 939 opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 940         RSA *rsa;
 941         isc_region_t r;
 942         unsigned int e_bytes;
 943 #if USE_EVP
 944         EVP_PKEY *pkey;
 945 #endif
 946
 947         isc_buffer_remainingregion(data, &r);
 948         if (r.length == 0)
 949                 return (ISC_R_SUCCESS);
 950
 951         rsa = RSA_new();
 952         if (rsa == NULL)
 953                 return (dst__openssl_toresult(ISC_R_NOMEMORY));
 954         SET_FLAGS(rsa);
 955
 956         if (r.length < 1) {
 957                 RSA_free(rsa);
 958                 return (DST_R_INVALIDPUBLICKEY);
 959         }
 960         e_bytes = *r.base++;
 961         r.length--;
 962
 963         if (e_bytes == 0) {
 964                 if (r.length < 2) {
 965                         RSA_free(rsa);
 966                         return (DST_R_INVALIDPUBLICKEY);
 967                 }
 968                 e_bytes = ((*r.base++) << 8);
 969                 e_bytes += *r.base++;
 970                 r.length -= 2;
 971         }
 972
 973         if (r.length < e_bytes) {
 974                 RSA_free(rsa);
 975                 return (DST_R_INVALIDPUBLICKEY);
 976         }
 977         rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
 978         r.base += e_bytes;
 979         r.length -= e_bytes;
 980
 981         rsa->n = BN_bin2bn(r.base, r.length, NULL);
 982
 983         key->key_size = BN_num_bits(rsa->n);
 984
 985         isc_buffer_forward(data, r.length);
 986
 987 #if USE_EVP
 988         pkey = EVP_PKEY_new();
 989         if (pkey == NULL) {
 990                 RSA_free(rsa);
 991                 return (ISC_R_NOMEMORY);
 992         }
 993         if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
 994                 EVP_PKEY_free(pkey);
 995                 RSA_free(rsa);
 996                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
 997         }
 998         key->keydata.pkey = pkey;
 999         RSA_free(rsa);
1000 #else
1001         key->keydata.rsa = rsa;
1002 #endif
1003
1004         return (ISC_R_SUCCESS);
1005 }
1006
1007 static isc_result_t
1008 opensslrsa_tofile(const dst_key_t *key, const char *directory) {
1009         int i;
1010         RSA *rsa;
1011         dst_private_t priv;
1012         unsigned char *bufs[8];
1013         isc_result_t result;
1014
1015 #if USE_EVP
1016         if (key->keydata.pkey == NULL)
1017                 return (DST_R_NULLKEY);
1018         rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
1019         if (rsa == NULL)
1020                 return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1021 #else
1022         if (key->keydata.rsa == NULL)
1023                 return (DST_R_NULLKEY);
1024         rsa = key->keydata.rsa;
1025 #endif
1026
1027         memset(bufs, 0, sizeof(bufs));
1028         for (i = 0; i < 8; i++) {
1029                 bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
1030                 if (bufs[i] == NULL) {
1031                         result = ISC_R_NOMEMORY;
1032                         goto fail;
1033                 }
1034         }
1035
1036         i = 0;
1037
1038         priv.elements[i].tag = TAG_RSA_MODULUS;
1039         priv.elements[i].length = BN_num_bytes(rsa->n);
1040         BN_bn2bin(rsa->n, bufs[i]);
1041         priv.elements[i].data = bufs[i];
1042         i++;
1043
1044         priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
1045         priv.elements[i].length = BN_num_bytes(rsa->e);
1046         BN_bn2bin(rsa->e, bufs[i]);
1047         priv.elements[i].data = bufs[i];
1048         i++;
1049
1050         if (rsa->d != NULL) {
1051                 priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
1052                 priv.elements[i].length = BN_num_bytes(rsa->d);
1053                 BN_bn2bin(rsa->d, bufs[i]);
1054                 priv.elements[i].data = bufs[i];
1055                 i++;
1056         }
1057
1058         if (rsa->p != NULL) {
1059                 priv.elements[i].tag = TAG_RSA_PRIME1;
1060                 priv.elements[i].length = BN_num_bytes(rsa->p);
1061                 BN_bn2bin(rsa->p, bufs[i]);
1062                 priv.elements[i].data = bufs[i];
1063                 i++;
1064         }
1065
1066         if (rsa->q != NULL) {
1067                 priv.elements[i].tag = TAG_RSA_PRIME2;
1068                 priv.elements[i].length = BN_num_bytes(rsa->q);
1069                 BN_bn2bin(rsa->q, bufs[i]);
1070                 priv.elements[i].data = bufs[i];
1071                 i++;
1072         }
1073
1074         if (rsa->dmp1 != NULL) {
1075                 priv.elements[i].tag = TAG_RSA_EXPONENT1;
1076                 priv.elements[i].length = BN_num_bytes(rsa->dmp1);
1077                 BN_bn2bin(rsa->dmp1, bufs[i]);
1078                 priv.elements[i].data = bufs[i];
1079                 i++;
1080         }
1081
1082         if (rsa->dmq1 != NULL) {
1083                 priv.elements[i].tag = TAG_RSA_EXPONENT2;
1084                 priv.elements[i].length = BN_num_bytes(rsa->dmq1);
1085                 BN_bn2bin(rsa->dmq1, bufs[i]);
1086                 priv.elements[i].data = bufs[i];
1087                 i++;
1088         }
1089
1090         if (rsa->iqmp != NULL) {
1091                 priv.elements[i].tag = TAG_RSA_COEFFICIENT;
1092                 priv.elements[i].length = BN_num_bytes(rsa->iqmp);
1093                 BN_bn2bin(rsa->iqmp, bufs[i]);
1094                 priv.elements[i].data = bufs[i];
1095                 i++;
1096         }
1097
1098         if (key->engine != NULL) {
1099                 priv.elements[i].tag = TAG_RSA_ENGINE;
1100                 priv.elements[i].length = strlen(key->engine) + 1;
1101                 priv.elements[i].data = (unsigned char *)key->engine;
1102                 i++;
1103         }
1104
1105         if (key->label != NULL) {
1106                 priv.elements[i].tag = TAG_RSA_LABEL;
1107                 priv.elements[i].length = strlen(key->label) + 1;
1108                 priv.elements[i].data = (unsigned char *)key->label;
1109                 i++;
1110         }
1111
1112
1113         priv.nelements = i;
1114         result = dst__privstruct_writefile(key, &priv, directory);
1115  fail:
1116 #if USE_EVP
1117         RSA_free(rsa);
1118 #endif
1119         for (i = 0; i < 8; i++) {
1120                 if (bufs[i] == NULL)
1121                         break;
1122                 isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
1123         }
1124         return (result);
1125 }
1126
1127 static isc_result_t
1128 rsa_check(RSA *rsa, RSA *pub)
1129 {
1130         /* Public parameters should be the same but if they are not set
1131          * copy them from the public key. */
1132         if (pub != NULL) {
1133                 if (rsa->n != NULL) {
1134                         if (BN_cmp(rsa->n, pub->n) != 0)
1135                                 return (DST_R_INVALIDPRIVATEKEY);
1136                 } else {
1137                         rsa->n = pub->n;
1138                         pub->n = NULL;
1139                 }
1140                 if (rsa->e != NULL) {
1141                         if (BN_cmp(rsa->e, pub->e) != 0)
1142                                 return (DST_R_INVALIDPRIVATEKEY);
1143                 } else {
1144                         rsa->e = pub->e;
1145                         pub->e = NULL;
1146                 }
1147         }
1148         if (rsa->n == NULL || rsa->e == NULL)
1149                 return (DST_R_INVALIDPRIVATEKEY);
1150         return (ISC_R_SUCCESS);
1151 }
1152
1153 static isc_result_t
1154 opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1155         dst_private_t priv;
1156         isc_result_t ret;
1157         int i;
1158         RSA *rsa = NULL, *pubrsa = NULL;
1159 #ifdef USE_ENGINE
1160         ENGINE *e = NULL;
1161 #endif
1162         isc_mem_t *mctx = key->mctx;
1163         const char *engine = NULL, *label = NULL;
1164 #if defined(USE_ENGINE) || USE_EVP
1165         EVP_PKEY *pkey = NULL;
1166 #endif
1167
1168 #if USE_EVP
1169         if (pub != NULL && pub->keydata.pkey != NULL)
1170                 pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
1171 #else
1172         if (pub != NULL && pub->keydata.rsa != NULL) {
1173                 pubrsa = pub->keydata.rsa;
1174                 pub->keydata.rsa = NULL;
1175         }
1176 #endif
1177
1178         /* read private key file */
1179         ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1180         if (ret != ISC_R_SUCCESS)
1181                 goto err;
1182
1183         for (i = 0; i < priv.nelements; i++) {
1184                 switch (priv.elements[i].tag) {
1185                 case TAG_RSA_ENGINE:
1186                         engine = (char *)priv.elements[i].data;
1187                         break;
1188                 case TAG_RSA_LABEL:
1189                         label = (char *)priv.elements[i].data;
1190                         break;
1191                 default:
1192                         break;
1193                 }
1194         }
1195         /*
1196          * Is this key is stored in a HSM?
1197          * See if we can fetch it.
1198          */
1199         if (label != NULL) {
1200 #ifdef USE_ENGINE
1201                 if (engine == NULL)
1202                         DST_RET(DST_R_NOENGINE);
1203                 e = dst__openssl_getengine(engine);
1204                 if (e == NULL)
1205                         DST_RET(DST_R_NOENGINE);
1206                 pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1207                 if (pkey == NULL)
1208                         DST_RET(dst__openssl_toresult2(
1209                                         "ENGINE_load_private_key",
1210                                         ISC_R_NOTFOUND));
1211                 key->engine = isc_mem_strdup(key->mctx, engine);
1212                 if (key->engine == NULL)
1213                         DST_RET(ISC_R_NOMEMORY);
1214                 key->label = isc_mem_strdup(key->mctx, label);
1215                 if (key->label == NULL)
1216                         DST_RET(ISC_R_NOMEMORY);
1217                 rsa = EVP_PKEY_get1_RSA(pkey);
1218                 if (rsa == NULL)
1219                         DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1220                 if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1221                         DST_RET(DST_R_INVALIDPRIVATEKEY);
1222                 if (pubrsa != NULL)
1223                         RSA_free(pubrsa);
1224                 key->key_size = EVP_PKEY_bits(pkey);
1225 #if USE_EVP
1226                 key->keydata.pkey = pkey;
1227                 RSA_free(rsa);
1228 #else
1229                 key->keydata.rsa = rsa;
1230                 EVP_PKEY_free(pkey);
1231 #endif
1232                 dst__privstruct_free(&priv, mctx);
1233                 memset(&priv, 0, sizeof(priv));
1234                 return (ISC_R_SUCCESS);
1235 #else
1236                 DST_RET(DST_R_NOENGINE);
1237 #endif
1238         }
1239
1240         rsa = RSA_new();
1241         if (rsa == NULL)
1242                 DST_RET(ISC_R_NOMEMORY);
1243         SET_FLAGS(rsa);
1244
1245 #if USE_EVP
1246         pkey = EVP_PKEY_new();
1247         if (pkey == NULL)
1248                 DST_RET(ISC_R_NOMEMORY);
1249         if (!EVP_PKEY_set1_RSA(pkey, rsa))
1250                 DST_RET(ISC_R_FAILURE);
1251         key->keydata.pkey = pkey;
1252 #else
1253         key->keydata.rsa = rsa;
1254 #endif
1255
1256         for (i = 0; i < priv.nelements; i++) {
1257                 BIGNUM *bn;
1258                 switch (priv.elements[i].tag) {
1259                 case TAG_RSA_ENGINE:
1260                         continue;
1261                 case TAG_RSA_LABEL:
1262                         continue;
1263                 case TAG_RSA_PIN:
1264                         continue;
1265                 default:
1266                         bn = BN_bin2bn(priv.elements[i].data,
1267                                        priv.elements[i].length, NULL);
1268                         if (bn == NULL)
1269                                 DST_RET(ISC_R_NOMEMORY);
1270                 }
1271
1272                 switch (priv.elements[i].tag) {
1273                         case TAG_RSA_MODULUS:
1274                                 rsa->n = bn;
1275                                 break;
1276                         case TAG_RSA_PUBLICEXPONENT:
1277                                 rsa->e = bn;
1278                                 break;
1279                         case TAG_RSA_PRIVATEEXPONENT:
1280                                 rsa->d = bn;
1281                                 break;
1282                         case TAG_RSA_PRIME1:
1283                                 rsa->p = bn;
1284                                 break;
1285                         case TAG_RSA_PRIME2:
1286                                 rsa->q = bn;
1287                                 break;
1288                         case TAG_RSA_EXPONENT1:
1289                                 rsa->dmp1 = bn;
1290                                 break;
1291                         case TAG_RSA_EXPONENT2:
1292                                 rsa->dmq1 = bn;
1293                                 break;
1294                         case TAG_RSA_COEFFICIENT:
1295                                 rsa->iqmp = bn;
1296                                 break;
1297                 }
1298         }
1299         dst__privstruct_free(&priv, mctx);
1300         memset(&priv, 0, sizeof(priv));
1301
1302         if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1303                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1304         key->key_size = BN_num_bits(rsa->n);
1305         if (pubrsa != NULL)
1306                 RSA_free(pubrsa);
1307 #if USE_EVP
1308         RSA_free(rsa);
1309 #endif
1310
1311         return (ISC_R_SUCCESS);
1312
1313  err:
1314 #if USE_EVP
1315         if (pkey != NULL)
1316                 EVP_PKEY_free(pkey);
1317 #endif
1318         if (rsa != NULL)
1319                 RSA_free(rsa);
1320         if (pubrsa != NULL)
1321                 RSA_free(pubrsa);
1322         opensslrsa_destroy(key);
1323         dst__privstruct_free(&priv, mctx);
1324         memset(&priv, 0, sizeof(priv));
1325         return (ret);
1326 }
1327
1328 static isc_result_t
1329 opensslrsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
1330                      const char *pin)
1331 {
1332 #ifdef USE_ENGINE
1333         ENGINE *e = NULL;
1334         isc_result_t ret;
1335         EVP_PKEY *pkey = NULL;
1336         RSA *rsa = NULL, *pubrsa = NULL;
1337         char *colon;
1338
1339         UNUSED(pin);
1340
1341         if (engine == NULL)
1342                 DST_RET(DST_R_NOENGINE);
1343         e = dst__openssl_getengine(engine);
1344         if (e == NULL)
1345                 DST_RET(DST_R_NOENGINE);
1346         pkey = ENGINE_load_public_key(e, label, NULL, NULL);
1347         if (pkey != NULL) {
1348                 pubrsa = EVP_PKEY_get1_RSA(pkey);
1349                 EVP_PKEY_free(pkey);
1350                 if (pubrsa == NULL)
1351                         DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1352         }
1353         pkey = ENGINE_load_private_key(e, label, NULL, NULL);
1354         if (pkey == NULL)
1355                 DST_RET(dst__openssl_toresult2("ENGINE_load_private_key",
1356                                                ISC_R_NOTFOUND));
1357         if (engine != NULL) {
1358                 key->engine = isc_mem_strdup(key->mctx, engine);
1359                 if (key->engine == NULL)
1360                         DST_RET(ISC_R_NOMEMORY);
1361         } else {
1362                 key->engine = isc_mem_strdup(key->mctx, label);
1363                 if (key->engine == NULL)
1364                         DST_RET(ISC_R_NOMEMORY);
1365                 colon = strchr(key->engine, ':');
1366                 if (colon != NULL)
1367                         *colon = '\0';
1368         }
1369         key->label = isc_mem_strdup(key->mctx, label);
1370         if (key->label == NULL)
1371                 DST_RET(ISC_R_NOMEMORY);
1372         rsa = EVP_PKEY_get1_RSA(pkey);
1373         if (rsa == NULL)
1374                 DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
1375         if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
1376                 DST_RET(DST_R_INVALIDPRIVATEKEY);
1377         if (pubrsa != NULL)
1378                 RSA_free(pubrsa);
1379         key->key_size = EVP_PKEY_bits(pkey);
1380 #if USE_EVP
1381         key->keydata.pkey = pkey;
1382         RSA_free(rsa);
1383 #else
1384         key->keydata.rsa = rsa;
1385         EVP_PKEY_free(pkey);
1386 #endif
1387         return (ISC_R_SUCCESS);
1388
1389  err:
1390         if (rsa != NULL)
1391                 RSA_free(rsa);
1392         if (pubrsa != NULL)
1393                 RSA_free(pubrsa);
1394         if (pkey != NULL)
1395                 EVP_PKEY_free(pkey);
1396         return (ret);
1397 #else
1398         UNUSED(key);
1399         UNUSED(engine);
1400         UNUSED(label);
1401         UNUSED(pin);
1402         return(DST_R_NOENGINE);
1403 #endif
1404 }
1405
1406 static dst_func_t opensslrsa_functions = {
1407         opensslrsa_createctx,
1408         opensslrsa_destroyctx,
1409         opensslrsa_adddata,
1410         opensslrsa_sign,
1411         opensslrsa_verify,
1412         NULL, /*%< computesecret */
1413         opensslrsa_compare,
1414         NULL, /*%< paramcompare */
1415         opensslrsa_generate,
1416         opensslrsa_isprivate,
1417         opensslrsa_destroy,
1418         opensslrsa_todns,
1419         opensslrsa_fromdns,
1420         opensslrsa_tofile,
1421         opensslrsa_parse,
1422         NULL, /*%< cleanup */
1423         opensslrsa_fromlabel,
1424         NULL, /*%< dump */
1425         NULL, /*%< restore */
1426 };
1427
1428 isc_result_t
1429 dst__opensslrsa_init(dst_func_t **funcp, unsigned char algorithm) {
1430         REQUIRE(funcp != NULL);
1431
1432         if (*funcp == NULL) {
1433                 switch (algorithm) {
1434                 case DST_ALG_RSASHA256:
1435 #if defined(HAVE_EVP_SHA256) || !USE_EVP
1436                         *funcp = &opensslrsa_functions;
1437 #endif
1438                         break;
1439                 case DST_ALG_RSASHA512:
1440 #if defined(HAVE_EVP_SHA512) || !USE_EVP
1441                         *funcp = &opensslrsa_functions;
1442 #endif
1443                         break;
1444                 default:
1445                         *funcp = &opensslrsa_functions;
1446                         break;
1447                 }
1448         }
1449         return (ISC_R_SUCCESS);
1450 }
1451
1452 #else /* OPENSSL */
1453
1454 #include <isc/util.h>
1455
1456 EMPTY_TRANSLATION_UNIT
1457
1458 #endif /* OPENSSL */
1459 /*! \file */