sys/geom/eli/g_eli.c

   1 /*-
   2  * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/cons.h>
  33 #include <sys/kernel.h>
  34 #include <sys/linker.h>
  35 #include <sys/module.h>
  36 #include <sys/lock.h>
  37 #include <sys/mutex.h>
  38 #include <sys/bio.h>
  39 #include <sys/sbuf.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/malloc.h>
  42 #include <sys/eventhandler.h>
  43 #include <sys/kthread.h>
  44 #include <sys/proc.h>
  45 #include <sys/sched.h>
  46 #include <sys/smp.h>
  47 #include <sys/uio.h>
  48 #include <sys/vnode.h>
  49
  50 #include <vm/uma.h>
  51
  52 #include <geom/geom.h>
  53 #include <geom/eli/g_eli.h>
  54 #include <geom/eli/pkcs5v2.h>
  55
  56 FEATURE(geom_eli, "GEOM crypto module");
  57
  58 MALLOC_DEFINE(M_ELI, "eli data", "GEOM_ELI Data");
  59
  60 SYSCTL_DECL(_kern_geom);
  61 SYSCTL_NODE(_kern_geom, OID_AUTO, eli, CTLFLAG_RW, 0, "GEOM_ELI stuff");
  62 static int g_eli_version = G_ELI_VERSION;
  63 SYSCTL_INT(_kern_geom_eli, OID_AUTO, version, CTLFLAG_RD, &g_eli_version, 0,
  64     "GELI version");
  65 int g_eli_debug = 0;
  66 SYSCTL_INT(_kern_geom_eli, OID_AUTO, debug, CTLFLAG_RWTUN, &g_eli_debug, 0,
  67     "Debug level");
  68 static u_int g_eli_tries = 3;
  69 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, tries, CTLFLAG_RWTUN, &g_eli_tries, 0,
  70     "Number of tries for entering the passphrase");
  71 static u_int g_eli_visible_passphrase = GETS_NOECHO;
  72 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, visible_passphrase, CTLFLAG_RWTUN,
  73     &g_eli_visible_passphrase, 0,
  74     "Visibility of passphrase prompt (0 = invisible, 1 = visible, 2 = asterisk)");
  75 u_int g_eli_overwrites = G_ELI_OVERWRITES;
  76 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RWTUN, &g_eli_overwrites,
  77     0, "Number of times on-disk keys should be overwritten when destroying them");
  78 static u_int g_eli_threads = 0;
  79 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RWTUN, &g_eli_threads, 0,
  80     "Number of threads doing crypto work");
  81 u_int g_eli_batch = 0;
  82 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0,
  83     "Use crypto operations batching");
  84
  85 /*
  86  * Passphrase cached during boot, in order to be more user-friendly if
  87  * there are multiple providers using the same passphrase.
  88  */
  89 static char cached_passphrase[256];
  90 static u_int g_eli_boot_passcache = 1;
  91 TUNABLE_INT("kern.geom.eli.boot_passcache", &g_eli_boot_passcache);
  92 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, boot_passcache, CTLFLAG_RD,
  93     &g_eli_boot_passcache, 0,
  94     "Passphrases are cached during boot process for possible reuse");
  95 static void
  96 fetch_loader_passphrase(void * dummy)
  97 {
  98         char * env_passphrase;
  99
 100         KASSERT(dynamic_kenv, ("need dynamic kenv"));
 101
 102         if ((env_passphrase = kern_getenv("kern.geom.eli.passphrase")) != NULL) {
 103                 /* Extract passphrase from the environment. */
 104                 strlcpy(cached_passphrase, env_passphrase,
 105                     sizeof(cached_passphrase));
 106                 freeenv(env_passphrase);
 107
 108                 /* Wipe the passphrase from the environment. */
 109                 kern_unsetenv("kern.geom.eli.passphrase");
 110         }
 111 }
 112 SYSINIT(geli_fetch_loader_passphrase, SI_SUB_KMEM + 1, SI_ORDER_ANY,
 113     fetch_loader_passphrase, NULL);
 114 static void
 115 zero_boot_passcache(void * dummy)
 116 {
 117
 118         memset(cached_passphrase, 0, sizeof(cached_passphrase));
 119 }
 120 EVENTHANDLER_DEFINE(mountroot, zero_boot_passcache, NULL, 0);
 121
 122 static eventhandler_tag g_eli_pre_sync = NULL;
 123
 124 static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
 125     struct g_geom *gp);
 126 static void g_eli_init(struct g_class *mp);
 127 static void g_eli_fini(struct g_class *mp);
 128
 129 static g_taste_t g_eli_taste;
 130 static g_dumpconf_t g_eli_dumpconf;
 131
 132 struct g_class g_eli_class = {
 133         .name = G_ELI_CLASS_NAME,
 134         .version = G_VERSION,
 135         .ctlreq = g_eli_config,
 136         .taste = g_eli_taste,
 137         .destroy_geom = g_eli_destroy_geom,
 138         .init = g_eli_init,
 139         .fini = g_eli_fini
 140 };
 141
 142
 143 /*
 144  * Code paths:
 145  * BIO_READ:
 146  *      g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
 147  * BIO_WRITE:
 148  *      g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
 149  */
 150
 151
 152 /*
 153  * EAGAIN from crypto(9) means, that we were probably balanced to another crypto
 154  * accelerator or something like this.
 155  * The function updates the SID and rerun the operation.
 156  */
 157 int
 158 g_eli_crypto_rerun(struct cryptop *crp)
 159 {
 160         struct g_eli_softc *sc;
 161         struct g_eli_worker *wr;
 162         struct bio *bp;
 163         int error;
 164
 165         bp = (struct bio *)crp->crp_opaque;
 166         sc = bp->bio_to->geom->softc;
 167         LIST_FOREACH(wr, &sc->sc_workers, w_next) {
 168                 if (wr->w_number == bp->bio_pflags)
 169                         break;
 170         }
 171         KASSERT(wr != NULL, ("Invalid worker (%u).", bp->bio_pflags));
 172         G_ELI_DEBUG(1, "Rerunning crypto %s request (sid: %ju -> %ju).",
 173             bp->bio_cmd == BIO_READ ? "READ" : "WRITE", (uintmax_t)wr->w_sid,
 174             (uintmax_t)crp->crp_sid);
 175         wr->w_sid = crp->crp_sid;
 176         crp->crp_etype = 0;
 177         error = crypto_dispatch(crp);
 178         if (error == 0)
 179                 return (0);
 180         G_ELI_DEBUG(1, "%s: crypto_dispatch() returned %d.", __func__, error);
 181         crp->crp_etype = error;
 182         return (error);
 183 }
 184
 185 /*
 186  * The function is called afer reading encrypted data from the provider.
 187  *
 188  * g_eli_start -> g_eli_crypto_read -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
 189  */
 190 void
 191 g_eli_read_done(struct bio *bp)
 192 {
 193         struct g_eli_softc *sc;
 194         struct bio *pbp;
 195
 196         G_ELI_LOGREQ(2, bp, "Request done.");
 197         pbp = bp->bio_parent;
 198         if (pbp->bio_error == 0 && bp->bio_error != 0)
 199                 pbp->bio_error = bp->bio_error;
 200         g_destroy_bio(bp);
 201         /*
 202          * Do we have all sectors already?
 203          */
 204         pbp->bio_inbed++;
 205         if (pbp->bio_inbed < pbp->bio_children)
 206                 return;
 207         sc = pbp->bio_to->geom->softc;
 208         if (pbp->bio_error != 0) {
 209                 G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,
 210                     pbp->bio_error);
 211                 pbp->bio_completed = 0;
 212                 if (pbp->bio_driver2 != NULL) {
 213                         free(pbp->bio_driver2, M_ELI);
 214                         pbp->bio_driver2 = NULL;
 215                 }
 216                 g_io_deliver(pbp, pbp->bio_error);
 217                 atomic_subtract_int(&sc->sc_inflight, 1);
 218                 return;
 219         }
 220         mtx_lock(&sc->sc_queue_mtx);
 221         bioq_insert_tail(&sc->sc_queue, pbp);
 222         mtx_unlock(&sc->sc_queue_mtx);
 223         wakeup(sc);
 224 }
 225
 226 /*
 227  * The function is called after we encrypt and write data.
 228  *
 229  * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
 230  */
 231 void
 232 g_eli_write_done(struct bio *bp)
 233 {
 234         struct g_eli_softc *sc;
 235         struct bio *pbp;
 236
 237         G_ELI_LOGREQ(2, bp, "Request done.");
 238         pbp = bp->bio_parent;
 239         if (pbp->bio_error == 0 && bp->bio_error != 0)
 240                 pbp->bio_error = bp->bio_error;
 241         g_destroy_bio(bp);
 242         /*
 243          * Do we have all sectors already?
 244          */
 245         pbp->bio_inbed++;
 246         if (pbp->bio_inbed < pbp->bio_children)
 247                 return;
 248         free(pbp->bio_driver2, M_ELI);
 249         pbp->bio_driver2 = NULL;
 250         if (pbp->bio_error != 0) {
 251                 G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,
 252                     pbp->bio_error);
 253                 pbp->bio_completed = 0;
 254         } else
 255                 pbp->bio_completed = pbp->bio_length;
 256
 257         /*
 258          * Write is finished, send it up.
 259          */
 260         sc = pbp->bio_to->geom->softc;
 261         g_io_deliver(pbp, pbp->bio_error);
 262         atomic_subtract_int(&sc->sc_inflight, 1);
 263 }
 264
 265 /*
 266  * This function should never be called, but GEOM made as it set ->orphan()
 267  * method for every geom.
 268  */
 269 static void
 270 g_eli_orphan_spoil_assert(struct g_consumer *cp)
 271 {
 272
 273         panic("Function %s() called for %s.", __func__, cp->geom->name);
 274 }
 275
 276 static void
 277 g_eli_orphan(struct g_consumer *cp)
 278 {
 279         struct g_eli_softc *sc;
 280
 281         g_topology_assert();
 282         sc = cp->geom->softc;
 283         if (sc == NULL)
 284                 return;
 285         g_eli_destroy(sc, TRUE);
 286 }
 287
 288 /*
 289  * BIO_READ:
 290  *      G_ELI_START -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
 291  * BIO_WRITE:
 292  *      G_ELI_START -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
 293  */
 294 static void
 295 g_eli_start(struct bio *bp)
 296 {
 297         struct g_eli_softc *sc;
 298         struct g_consumer *cp;
 299         struct bio *cbp;
 300
 301         sc = bp->bio_to->geom->softc;
 302         KASSERT(sc != NULL,
 303             ("Provider's error should be set (error=%d)(device=%s).",
 304             bp->bio_to->error, bp->bio_to->name));
 305         G_ELI_LOGREQ(2, bp, "Request received.");
 306
 307         switch (bp->bio_cmd) {
 308         case BIO_READ:
 309         case BIO_WRITE:
 310         case BIO_GETATTR:
 311         case BIO_FLUSH:
 312         case BIO_ZONE:
 313                 break;
 314         case BIO_DELETE:
 315                 /*
 316                  * If the user hasn't set the NODELETE flag, we just pass
 317                  * it down the stack and let the layers beneath us do (or
 318                  * not) whatever they do with it.  If they have, we
 319                  * reject it.  A possible extension would be an
 320                  * additional flag to take it as a hint to shred the data
 321                  * with [multiple?] overwrites.
 322                  */
 323                 if (!(sc->sc_flags & G_ELI_FLAG_NODELETE))
 324                         break;
 325         default:
 326                 g_io_deliver(bp, EOPNOTSUPP);
 327                 return;
 328         }
 329         cbp = g_clone_bio(bp);
 330         if (cbp == NULL) {
 331                 g_io_deliver(bp, ENOMEM);
 332                 return;
 333         }
 334         bp->bio_driver1 = cbp;
 335         bp->bio_pflags = G_ELI_NEW_BIO;
 336         switch (bp->bio_cmd) {
 337         case BIO_READ:
 338                 if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) {
 339                         g_eli_crypto_read(sc, bp, 0);
 340                         break;
 341                 }
 342                 /* FALLTHROUGH */
 343         case BIO_WRITE:
 344                 mtx_lock(&sc->sc_queue_mtx);
 345                 bioq_insert_tail(&sc->sc_queue, bp);
 346                 mtx_unlock(&sc->sc_queue_mtx);
 347                 wakeup(sc);
 348                 break;
 349         case BIO_GETATTR:
 350         case BIO_FLUSH:
 351         case BIO_DELETE:
 352         case BIO_ZONE:
 353                 cbp->bio_done = g_std_done;
 354                 cp = LIST_FIRST(&sc->sc_geom->consumer);
 355                 cbp->bio_to = cp->provider;
 356                 G_ELI_LOGREQ(2, cbp, "Sending request.");
 357                 g_io_request(cbp, cp);
 358                 break;
 359         }
 360 }
 361
 362 static int
 363 g_eli_newsession(struct g_eli_worker *wr)
 364 {
 365         struct g_eli_softc *sc;
 366         struct cryptoini crie, cria;
 367         int error;
 368
 369         sc = wr->w_softc;
 370
 371         bzero(&crie, sizeof(crie));
 372         crie.cri_alg = sc->sc_ealgo;
 373         crie.cri_klen = sc->sc_ekeylen;
 374         if (sc->sc_ealgo == CRYPTO_AES_XTS)
 375                 crie.cri_klen <<= 1;
 376         if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
 377                 crie.cri_key = g_eli_key_hold(sc, 0,
 378                     LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize);
 379         } else {
 380                 crie.cri_key = sc->sc_ekey;
 381         }
 382         if (sc->sc_flags & G_ELI_FLAG_AUTH) {
 383                 bzero(&cria, sizeof(cria));
 384                 cria.cri_alg = sc->sc_aalgo;
 385                 cria.cri_klen = sc->sc_akeylen;
 386                 cria.cri_key = sc->sc_akey;
 387                 crie.cri_next = &cria;
 388         }
 389
 390         switch (sc->sc_crypto) {
 391         case G_ELI_CRYPTO_SW:
 392                 error = crypto_newsession(&wr->w_sid, &crie,
 393                     CRYPTOCAP_F_SOFTWARE);
 394                 break;
 395         case G_ELI_CRYPTO_HW:
 396                 error = crypto_newsession(&wr->w_sid, &crie,
 397                     CRYPTOCAP_F_HARDWARE);
 398                 break;
 399         case G_ELI_CRYPTO_UNKNOWN:
 400                 error = crypto_newsession(&wr->w_sid, &crie,
 401                     CRYPTOCAP_F_HARDWARE);
 402                 if (error == 0) {
 403                         mtx_lock(&sc->sc_queue_mtx);
 404                         if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
 405                                 sc->sc_crypto = G_ELI_CRYPTO_HW;
 406                         mtx_unlock(&sc->sc_queue_mtx);
 407                 } else {
 408                         error = crypto_newsession(&wr->w_sid, &crie,
 409                             CRYPTOCAP_F_SOFTWARE);
 410                         mtx_lock(&sc->sc_queue_mtx);
 411                         if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
 412                                 sc->sc_crypto = G_ELI_CRYPTO_SW;
 413                         mtx_unlock(&sc->sc_queue_mtx);
 414                 }
 415                 break;
 416         default:
 417                 panic("%s: invalid condition", __func__);
 418         }
 419
 420         if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0)
 421                 g_eli_key_drop(sc, crie.cri_key);
 422
 423         return (error);
 424 }
 425
 426 static void
 427 g_eli_freesession(struct g_eli_worker *wr)
 428 {
 429
 430         crypto_freesession(wr->w_sid);
 431 }
 432
 433 static void
 434 g_eli_cancel(struct g_eli_softc *sc)
 435 {
 436         struct bio *bp;
 437
 438         mtx_assert(&sc->sc_queue_mtx, MA_OWNED);
 439
 440         while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL) {
 441                 KASSERT(bp->bio_pflags == G_ELI_NEW_BIO,
 442                     ("Not new bio when canceling (bp=%p).", bp));
 443                 g_io_deliver(bp, ENXIO);
 444         }
 445 }
 446
 447 static struct bio *
 448 g_eli_takefirst(struct g_eli_softc *sc)
 449 {
 450         struct bio *bp;
 451
 452         mtx_assert(&sc->sc_queue_mtx, MA_OWNED);
 453
 454         if (!(sc->sc_flags & G_ELI_FLAG_SUSPEND))
 455                 return (bioq_takefirst(&sc->sc_queue));
 456         /*
 457          * Device suspended, so we skip new I/O requests.
 458          */
 459         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 460                 if (bp->bio_pflags != G_ELI_NEW_BIO)
 461                         break;
 462         }
 463         if (bp != NULL)
 464                 bioq_remove(&sc->sc_queue, bp);
 465         return (bp);
 466 }
 467
 468 /*
 469  * This is the main function for kernel worker thread when we don't have
 470  * hardware acceleration and we have to do cryptography in software.
 471  * Dedicated thread is needed, so we don't slow down g_up/g_down GEOM
 472  * threads with crypto work.
 473  */
 474 static void
 475 g_eli_worker(void *arg)
 476 {
 477         struct g_eli_softc *sc;
 478         struct g_eli_worker *wr;
 479         struct bio *bp;
 480         int error;
 481
 482         wr = arg;
 483         sc = wr->w_softc;
 484 #ifdef EARLY_AP_STARTUP
 485         MPASS(!sc->sc_cpubind || smp_started);
 486 #elif defined(SMP)
 487         /* Before sched_bind() to a CPU, wait for all CPUs to go on-line. */
 488         if (sc->sc_cpubind) {
 489                 while (!smp_started)
 490                         tsleep(wr, 0, "geli:smp", hz / 4);
 491         }
 492 #endif
 493         thread_lock(curthread);
 494         sched_prio(curthread, PUSER);
 495         if (sc->sc_cpubind)
 496                 sched_bind(curthread, wr->w_number % mp_ncpus);
 497         thread_unlock(curthread);
 498
 499         G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
 500
 501         for (;;) {
 502                 mtx_lock(&sc->sc_queue_mtx);
 503 again:
 504                 bp = g_eli_takefirst(sc);
 505                 if (bp == NULL) {
 506                         if (sc->sc_flags & G_ELI_FLAG_DESTROY) {
 507                                 g_eli_cancel(sc);
 508                                 LIST_REMOVE(wr, w_next);
 509                                 g_eli_freesession(wr);
 510                                 free(wr, M_ELI);
 511                                 G_ELI_DEBUG(1, "Thread %s exiting.",
 512                                     curthread->td_proc->p_comm);
 513                                 wakeup(&sc->sc_workers);
 514                                 mtx_unlock(&sc->sc_queue_mtx);
 515                                 kproc_exit(0);
 516                         }
 517                         while (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
 518                                 if (sc->sc_inflight > 0) {
 519                                         G_ELI_DEBUG(0, "inflight=%d",
 520                                             sc->sc_inflight);
 521                                         /*
 522                                          * We still have inflight BIOs, so
 523                                          * sleep and retry.
 524                                          */
 525                                         msleep(sc, &sc->sc_queue_mtx, PRIBIO,
 526                                             "geli:inf", hz / 5);
 527                                         goto again;
 528                                 }
 529                                 /*
 530                                  * Suspend requested, mark the worker as
 531                                  * suspended and go to sleep.
 532                                  */
 533                                 if (wr->w_active) {
 534                                         g_eli_freesession(wr);
 535                                         wr->w_active = FALSE;
 536                                 }
 537                                 wakeup(&sc->sc_workers);
 538                                 msleep(sc, &sc->sc_queue_mtx, PRIBIO,
 539                                     "geli:suspend", 0);
 540                                 if (!wr->w_active &&
 541                                     !(sc->sc_flags & G_ELI_FLAG_SUSPEND)) {
 542                                         error = g_eli_newsession(wr);
 543                                         KASSERT(error == 0,
 544                                             ("g_eli_newsession() failed on resume (error=%d)",
 545                                             error));
 546                                         wr->w_active = TRUE;
 547                                 }
 548                                 goto again;
 549                         }
 550                         msleep(sc, &sc->sc_queue_mtx, PDROP, "geli:w", 0);
 551                         continue;
 552                 }
 553                 if (bp->bio_pflags == G_ELI_NEW_BIO)
 554                         atomic_add_int(&sc->sc_inflight, 1);
 555                 mtx_unlock(&sc->sc_queue_mtx);
 556                 if (bp->bio_pflags == G_ELI_NEW_BIO) {
 557                         bp->bio_pflags = 0;
 558                         if (sc->sc_flags & G_ELI_FLAG_AUTH) {
 559                                 if (bp->bio_cmd == BIO_READ)
 560                                         g_eli_auth_read(sc, bp);
 561                                 else
 562                                         g_eli_auth_run(wr, bp);
 563                         } else {
 564                                 if (bp->bio_cmd == BIO_READ)
 565                                         g_eli_crypto_read(sc, bp, 1);
 566                                 else
 567                                         g_eli_crypto_run(wr, bp);
 568                         }
 569                 } else {
 570                         if (sc->sc_flags & G_ELI_FLAG_AUTH)
 571                                 g_eli_auth_run(wr, bp);
 572                         else
 573                                 g_eli_crypto_run(wr, bp);
 574                 }
 575         }
 576 }
 577
 578 int
 579 g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
 580     struct g_eli_metadata *md)
 581 {
 582         struct g_geom *gp;
 583         struct g_consumer *cp;
 584         u_char *buf = NULL;
 585         int error;
 586
 587         g_topology_assert();
 588
 589         gp = g_new_geomf(mp, "eli:taste");
 590         gp->start = g_eli_start;
 591         gp->access = g_std_access;
 592         /*
 593          * g_eli_read_metadata() is always called from the event thread.
 594          * Our geom is created and destroyed in the same event, so there
 595          * could be no orphan nor spoil event in the meantime.
 596          */
 597         gp->orphan = g_eli_orphan_spoil_assert;
 598         gp->spoiled = g_eli_orphan_spoil_assert;
 599         cp = g_new_consumer(gp);
 600         error = g_attach(cp, pp);
 601         if (error != 0)
 602                 goto end;
 603         error = g_access(cp, 1, 0, 0);
 604         if (error != 0)
 605                 goto end;
 606         g_topology_unlock();
 607         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
 608             &error);
 609         g_topology_lock();
 610         if (buf == NULL)
 611                 goto end;
 612         error = eli_metadata_decode(buf, md);
 613         if (error != 0)
 614                 goto end;
 615         /* Metadata was read and decoded successfully. */
 616 end:
 617         if (buf != NULL)
 618                 g_free(buf);
 619         if (cp->provider != NULL) {
 620                 if (cp->acr == 1)
 621                         g_access(cp, -1, 0, 0);
 622                 g_detach(cp);
 623         }
 624         g_destroy_consumer(cp);
 625         g_destroy_geom(gp);
 626         return (error);
 627 }
 628
 629 /*
 630  * The function is called when we had last close on provider and user requested
 631  * to close it when this situation occur.
 632  */
 633 static void
 634 g_eli_last_close(void *arg, int flags __unused)
 635 {
 636         struct g_geom *gp;
 637         char gpname[64];
 638         int error;
 639
 640         g_topology_assert();
 641         gp = arg;
 642         strlcpy(gpname, gp->name, sizeof(gpname));
 643         error = g_eli_destroy(gp->softc, TRUE);
 644         KASSERT(error == 0, ("Cannot detach %s on last close (error=%d).",
 645             gpname, error));
 646         G_ELI_DEBUG(0, "Detached %s on last close.", gpname);
 647 }
 648
 649 int
 650 g_eli_access(struct g_provider *pp, int dr, int dw, int de)
 651 {
 652         struct g_eli_softc *sc;
 653         struct g_geom *gp;
 654
 655         gp = pp->geom;
 656         sc = gp->softc;
 657
 658         if (dw > 0) {
 659                 if (sc->sc_flags & G_ELI_FLAG_RO) {
 660                         /* Deny write attempts. */
 661                         return (EROFS);
 662                 }
 663                 /* Someone is opening us for write, we need to remember that. */
 664                 sc->sc_flags |= G_ELI_FLAG_WOPEN;
 665                 return (0);
 666         }
 667         /* Is this the last close? */
 668         if (pp->acr + dr > 0 || pp->acw + dw > 0 || pp->ace + de > 0)
 669                 return (0);
 670
 671         /*
 672          * Automatically detach on last close if requested.
 673          */
 674         if ((sc->sc_flags & G_ELI_FLAG_RW_DETACH) ||
 675             (sc->sc_flags & G_ELI_FLAG_WOPEN)) {
 676                 g_post_event(g_eli_last_close, gp, M_WAITOK, NULL);
 677         }
 678         return (0);
 679 }
 680
 681 static int
 682 g_eli_cpu_is_disabled(int cpu)
 683 {
 684 #ifdef SMP
 685         return (CPU_ISSET(cpu, &hlt_cpus_mask));
 686 #else
 687         return (0);
 688 #endif
 689 }
 690
 691 struct g_geom *
 692 g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 693     const struct g_eli_metadata *md, const u_char *mkey, int nkey)
 694 {
 695         struct g_eli_softc *sc;
 696         struct g_eli_worker *wr;
 697         struct g_geom *gp;
 698         struct g_provider *pp;
 699         struct g_consumer *cp;
 700         u_int i, threads;
 701         int error;
 702
 703         G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX);
 704
 705         gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX);
 706         sc = malloc(sizeof(*sc), M_ELI, M_WAITOK | M_ZERO);
 707         gp->start = g_eli_start;
 708         /*
 709          * Spoiling can happen even though we have the provider open
 710          * exclusively, e.g. through media change events.
 711          */
 712         gp->spoiled = g_eli_orphan;
 713         gp->orphan = g_eli_orphan;
 714         gp->dumpconf = g_eli_dumpconf;
 715         /*
 716          * If detach-on-last-close feature is not enabled and we don't operate
 717          * on read-only provider, we can simply use g_std_access().
 718          */
 719         if (md->md_flags & (G_ELI_FLAG_WO_DETACH | G_ELI_FLAG_RO))
 720                 gp->access = g_eli_access;
 721         else
 722                 gp->access = g_std_access;
 723
 724         eli_metadata_softc(sc, md, bpp->sectorsize, bpp->mediasize);
 725         sc->sc_nkey = nkey;
 726
 727         gp->softc = sc;
 728         sc->sc_geom = gp;
 729
 730         bioq_init(&sc->sc_queue);
 731         mtx_init(&sc->sc_queue_mtx, "geli:queue", NULL, MTX_DEF);
 732         mtx_init(&sc->sc_ekeys_lock, "geli:ekeys", NULL, MTX_DEF);
 733
 734         pp = NULL;
 735         cp = g_new_consumer(gp);
 736         error = g_attach(cp, bpp);
 737         if (error != 0) {
 738                 if (req != NULL) {
 739                         gctl_error(req, "Cannot attach to %s (error=%d).",
 740                             bpp->name, error);
 741                 } else {
 742                         G_ELI_DEBUG(1, "Cannot attach to %s (error=%d).",
 743                             bpp->name, error);
 744                 }
 745                 goto failed;
 746         }
 747         /*
 748          * Keep provider open all the time, so we can run critical tasks,
 749          * like Master Keys deletion, without wondering if we can open
 750          * provider or not.
 751          * We don't open provider for writing only when user requested read-only
 752          * access.
 753          */
 754         if (sc->sc_flags & G_ELI_FLAG_RO)
 755                 error = g_access(cp, 1, 0, 1);
 756         else
 757                 error = g_access(cp, 1, 1, 1);
 758         if (error != 0) {
 759                 if (req != NULL) {
 760                         gctl_error(req, "Cannot access %s (error=%d).",
 761                             bpp->name, error);
 762                 } else {
 763                         G_ELI_DEBUG(1, "Cannot access %s (error=%d).",
 764                             bpp->name, error);
 765                 }
 766                 goto failed;
 767         }
 768
 769         /*
 770          * Remember the keys in our softc structure.
 771          */
 772         g_eli_mkey_propagate(sc, mkey);
 773
 774         LIST_INIT(&sc->sc_workers);
 775
 776         threads = g_eli_threads;
 777         if (threads == 0)
 778                 threads = mp_ncpus;
 779         sc->sc_cpubind = (mp_ncpus > 1 && threads == mp_ncpus);
 780         for (i = 0; i < threads; i++) {
 781                 if (g_eli_cpu_is_disabled(i)) {
 782                         G_ELI_DEBUG(1, "%s: CPU %u disabled, skipping.",
 783                             bpp->name, i);
 784                         continue;
 785                 }
 786                 wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
 787                 wr->w_softc = sc;
 788                 wr->w_number = i;
 789                 wr->w_active = TRUE;
 790
 791                 error = g_eli_newsession(wr);
 792                 if (error != 0) {
 793                         free(wr, M_ELI);
 794                         if (req != NULL) {
 795                                 gctl_error(req, "Cannot set up crypto session "
 796                                     "for %s (error=%d).", bpp->name, error);
 797                         } else {
 798                                 G_ELI_DEBUG(1, "Cannot set up crypto session "
 799                                     "for %s (error=%d).", bpp->name, error);
 800                         }
 801                         goto failed;
 802                 }
 803
 804                 error = kproc_create(g_eli_worker, wr, &wr->w_proc, 0, 0,
 805                     "g_eli[%u] %s", i, bpp->name);
 806                 if (error != 0) {
 807                         g_eli_freesession(wr);
 808                         free(wr, M_ELI);
 809                         if (req != NULL) {
 810                                 gctl_error(req, "Cannot create kernel thread "
 811                                     "for %s (error=%d).", bpp->name, error);
 812                         } else {
 813                                 G_ELI_DEBUG(1, "Cannot create kernel thread "
 814                                     "for %s (error=%d).", bpp->name, error);
 815                         }
 816                         goto failed;
 817                 }
 818                 LIST_INSERT_HEAD(&sc->sc_workers, wr, w_next);
 819         }
 820
 821         /*
 822          * Create decrypted provider.
 823          */
 824         pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
 825         pp->mediasize = sc->sc_mediasize;
 826         pp->sectorsize = sc->sc_sectorsize;
 827
 828         g_error_provider(pp, 0);
 829
 830         G_ELI_DEBUG(0, "Device %s created.", pp->name);
 831         G_ELI_DEBUG(0, "Encryption: %s %u", g_eli_algo2str(sc->sc_ealgo),
 832             sc->sc_ekeylen);
 833         if (sc->sc_flags & G_ELI_FLAG_AUTH)
 834                 G_ELI_DEBUG(0, " Integrity: %s", g_eli_algo2str(sc->sc_aalgo));
 835         G_ELI_DEBUG(0, "    Crypto: %s",
 836             sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
 837         return (gp);
 838 failed:
 839         mtx_lock(&sc->sc_queue_mtx);
 840         sc->sc_flags |= G_ELI_FLAG_DESTROY;
 841         wakeup(sc);
 842         /*
 843          * Wait for kernel threads self destruction.
 844          */
 845         while (!LIST_EMPTY(&sc->sc_workers)) {
 846                 msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
 847                     "geli:destroy", 0);
 848         }
 849         mtx_destroy(&sc->sc_queue_mtx);
 850         if (cp->provider != NULL) {
 851                 if (cp->acr == 1)
 852                         g_access(cp, -1, -1, -1);
 853                 g_detach(cp);
 854         }
 855         g_destroy_consumer(cp);
 856         g_destroy_geom(gp);
 857         g_eli_key_destroy(sc);
 858         bzero(sc, sizeof(*sc));
 859         free(sc, M_ELI);
 860         return (NULL);
 861 }
 862
 863 int
 864 g_eli_destroy(struct g_eli_softc *sc, boolean_t force)
 865 {
 866         struct g_geom *gp;
 867         struct g_provider *pp;
 868
 869         g_topology_assert();
 870
 871         if (sc == NULL)
 872                 return (ENXIO);
 873
 874         gp = sc->sc_geom;
 875         pp = LIST_FIRST(&gp->provider);
 876         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
 877                 if (force) {
 878                         G_ELI_DEBUG(1, "Device %s is still open, so it "
 879                             "cannot be definitely removed.", pp->name);
 880                         sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
 881                         gp->access = g_eli_access;
 882                         g_wither_provider(pp, ENXIO);
 883                         return (EBUSY);
 884                 } else {
 885                         G_ELI_DEBUG(1,
 886                             "Device %s is still open (r%dw%de%d).", pp->name,
 887                             pp->acr, pp->acw, pp->ace);
 888                         return (EBUSY);
 889                 }
 890         }
 891
 892         mtx_lock(&sc->sc_queue_mtx);
 893         sc->sc_flags |= G_ELI_FLAG_DESTROY;
 894         wakeup(sc);
 895         while (!LIST_EMPTY(&sc->sc_workers)) {
 896                 msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
 897                     "geli:destroy", 0);
 898         }
 899         mtx_destroy(&sc->sc_queue_mtx);
 900         gp->softc = NULL;
 901         g_eli_key_destroy(sc);
 902         bzero(sc, sizeof(*sc));
 903         free(sc, M_ELI);
 904
 905         if (pp == NULL || (pp->acr == 0 && pp->acw == 0 && pp->ace == 0))
 906                 G_ELI_DEBUG(0, "Device %s destroyed.", gp->name);
 907         g_wither_geom_close(gp, ENXIO);
 908
 909         return (0);
 910 }
 911
 912 static int
 913 g_eli_destroy_geom(struct gctl_req *req __unused,
 914     struct g_class *mp __unused, struct g_geom *gp)
 915 {
 916         struct g_eli_softc *sc;
 917
 918         sc = gp->softc;
 919         return (g_eli_destroy(sc, FALSE));
 920 }
 921
 922 static int
 923 g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider)
 924 {
 925         u_char *keyfile, *data;
 926         char *file, name[64];
 927         size_t size;
 928         int i;
 929
 930         for (i = 0; ; i++) {
 931                 snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
 932                 keyfile = preload_search_by_type(name);
 933                 if (keyfile == NULL && i == 0) {
 934                         /*
 935                          * If there is only one keyfile, allow simpler name.
 936                          */
 937                         snprintf(name, sizeof(name), "%s:geli_keyfile", provider);
 938                         keyfile = preload_search_by_type(name);
 939                 }
 940                 if (keyfile == NULL)
 941                         return (i);     /* Return number of loaded keyfiles. */
 942                 data = preload_fetch_addr(keyfile);
 943                 if (data == NULL) {
 944                         G_ELI_DEBUG(0, "Cannot find key file data for %s.",
 945                             name);
 946                         return (0);
 947                 }
 948                 size = preload_fetch_size(keyfile);
 949                 if (size == 0) {
 950                         G_ELI_DEBUG(0, "Cannot find key file size for %s.",
 951                             name);
 952                         return (0);
 953                 }
 954                 file = preload_search_info(keyfile, MODINFO_NAME);
 955                 if (file == NULL) {
 956                         G_ELI_DEBUG(0, "Cannot find key file name for %s.",
 957                             name);
 958                         return (0);
 959                 }
 960                 G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file,
 961                     provider, name);
 962                 g_eli_crypto_hmac_update(ctx, data, size);
 963         }
 964 }
 965
 966 static void
 967 g_eli_keyfiles_clear(const char *provider)
 968 {
 969         u_char *keyfile, *data;
 970         char name[64];
 971         size_t size;
 972         int i;
 973
 974         for (i = 0; ; i++) {
 975                 snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
 976                 keyfile = preload_search_by_type(name);
 977                 if (keyfile == NULL)
 978                         return;
 979                 data = preload_fetch_addr(keyfile);
 980                 size = preload_fetch_size(keyfile);
 981                 if (data != NULL && size != 0)
 982                         bzero(data, size);
 983         }
 984 }
 985
 986 /*
 987  * Tasting is only made on boot.
 988  * We detect providers which should be attached before root is mounted.
 989  */
 990 static struct g_geom *
 991 g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 992 {
 993         struct g_eli_metadata md;
 994         struct g_geom *gp;
 995         struct hmac_ctx ctx;
 996         char passphrase[256];
 997         u_char key[G_ELI_USERKEYLEN], mkey[G_ELI_DATAIVKEYLEN];
 998         u_int i, nkey, nkeyfiles, tries;
 999         int error;
1000
1001         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
1002         g_topology_assert();
1003
1004         if (root_mounted() || g_eli_tries == 0)
1005                 return (NULL);
1006
1007         G_ELI_DEBUG(3, "Tasting %s.", pp->name);
1008
1009         error = g_eli_read_metadata(mp, pp, &md);
1010         if (error != 0)
1011                 return (NULL);
1012         gp = NULL;
1013
1014         if (strcmp(md.md_magic, G_ELI_MAGIC) != 0)
1015                 return (NULL);
1016         if (md.md_version > G_ELI_VERSION) {
1017                 printf("geom_eli.ko module is too old to handle %s.\n",
1018                     pp->name);
1019                 return (NULL);
1020         }
1021         if (md.md_provsize != pp->mediasize)
1022                 return (NULL);
1023         /* Should we attach it on boot? */
1024         if (!(md.md_flags & G_ELI_FLAG_BOOT))
1025                 return (NULL);
1026         if (md.md_keys == 0x00) {
1027                 G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
1028                 return (NULL);
1029         }
1030         if (md.md_iterations == -1) {
1031                 /* If there is no passphrase, we try only once. */
1032                 tries = 1;
1033         } else {
1034                 /* Ask for the passphrase no more than g_eli_tries times. */
1035                 tries = g_eli_tries;
1036         }
1037
1038         /*
1039          * CyberLeo: Attempt to decrypt the provider using only the keys, to avoid kern/193624
1040          */
1041         g_eli_crypto_hmac_init(&ctx, NULL, 0);
1042         if (0 < g_eli_keyfiles_load(&ctx, pp->name)) {
1043                 g_eli_crypto_hmac_final(&ctx, key, 0);
1044                 error = g_eli_mkey_decrypt(&md, key, mkey, &nkey);
1045                 bzero(key, sizeof(key));
1046                 if (error == -1) {
1047                         G_ELI_DEBUG(0,
1048                             "Wrong key for %s.",
1049                             pp->name);
1050                 } else if (error > 0) {
1051                         G_ELI_DEBUG(0,
1052                             "Cannot decrypt Master Key for %s (error=%d).",
1053                             pp->name, error);
1054                         g_eli_keyfiles_clear(pp->name);
1055                         return (NULL);
1056                 } else {
1057                         G_ELI_DEBUG(0,
1058                             "Decrypted master key for %s without passphrase.",
1059                             pp->name);
1060                         tries = 0;
1061                 }
1062         } else {
1063                 /* Cleanup */
1064                 g_eli_crypto_hmac_final(&ctx, key, 0);
1065                 bzero(key, sizeof(key));
1066         }
1067
1068         for (i = 0; i <= tries; i++) {
1069                 /* Avoid running loop at all if there are no tries */
1070                 if (tries < 1)
1071                         break;
1072
1073                 g_eli_crypto_hmac_init(&ctx, NULL, 0);
1074
1075                 /*
1076                  * Load all key files.
1077                  */
1078                 nkeyfiles = g_eli_keyfiles_load(&ctx, pp->name);
1079
1080                 if (nkeyfiles == 0 && md.md_iterations == -1) {
1081                         /*
1082                          * No key files and no passphrase, something is
1083                          * definitely wrong here.
1084                          * geli(8) doesn't allow for such situation, so assume
1085                          * that there was really no passphrase and in that case
1086                          * key files are no properly defined in loader.conf.
1087                          */
1088                         G_ELI_DEBUG(0,
1089                             "Found no key files in loader.conf for %s.",
1090                             pp->name);
1091                         return (NULL);
1092                 }
1093
1094                 /* Ask for the passphrase if defined. */
1095                 if (md.md_iterations >= 0) {
1096                         /* Try first with cached passphrase. */
1097                         if (i == 0) {
1098                                 if (!g_eli_boot_passcache)
1099                                         continue;
1100                                 memcpy(passphrase, cached_passphrase,
1101                                     sizeof(passphrase));
1102                         } else {
1103                                 printf("Enter passphrase for %s: ", pp->name);
1104                                 cngets(passphrase, sizeof(passphrase),
1105                                     g_eli_visible_passphrase);
1106                                 memcpy(cached_passphrase, passphrase,
1107                                     sizeof(passphrase));
1108                         }
1109                 }
1110
1111                 /*
1112                  * Prepare Derived-Key from the user passphrase.
1113                  */
1114                 if (md.md_iterations == 0) {
1115                         g_eli_crypto_hmac_update(&ctx, md.md_salt,
1116                             sizeof(md.md_salt));
1117                         g_eli_crypto_hmac_update(&ctx, passphrase,
1118                             strlen(passphrase));
1119                         bzero(passphrase, sizeof(passphrase));
1120                 } else if (md.md_iterations > 0) {
1121                         u_char dkey[G_ELI_USERKEYLEN];
1122
1123                         pkcs5v2_genkey(dkey, sizeof(dkey), md.md_salt,
1124                             sizeof(md.md_salt), passphrase, md.md_iterations);
1125                         bzero(passphrase, sizeof(passphrase));
1126                         g_eli_crypto_hmac_update(&ctx, dkey, sizeof(dkey));
1127                         bzero(dkey, sizeof(dkey));
1128                 }
1129
1130                 g_eli_crypto_hmac_final(&ctx, key, 0);
1131
1132                 /*
1133                  * Decrypt Master-Key.
1134                  */
1135                 error = g_eli_mkey_decrypt(&md, key, mkey, &nkey);
1136                 bzero(key, sizeof(key));
1137                 if (error == -1) {
1138                         if (i == tries) {
1139                                 G_ELI_DEBUG(0,
1140                                     "Wrong key for %s. No tries left.",
1141                                     pp->name);
1142                                 g_eli_keyfiles_clear(pp->name);
1143                                 return (NULL);
1144                         }
1145                         if (i > 0) {
1146                                 G_ELI_DEBUG(0,
1147                                     "Wrong key for %s. Tries left: %u.",
1148                                     pp->name, tries - i);
1149                         }
1150                         /* Try again. */
1151                         continue;
1152                 } else if (error > 0) {
1153                         G_ELI_DEBUG(0,
1154                             "Cannot decrypt Master Key for %s (error=%d).",
1155                             pp->name, error);
1156                         g_eli_keyfiles_clear(pp->name);
1157                         return (NULL);
1158                 }
1159                 g_eli_keyfiles_clear(pp->name);
1160                 G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name);
1161                 break;
1162         }
1163
1164         /*
1165          * We have correct key, let's attach provider.
1166          */
1167         gp = g_eli_create(NULL, mp, pp, &md, mkey, nkey);
1168         bzero(mkey, sizeof(mkey));
1169         bzero(&md, sizeof(md));
1170         if (gp == NULL) {
1171                 G_ELI_DEBUG(0, "Cannot create device %s%s.", pp->name,
1172                     G_ELI_SUFFIX);
1173                 return (NULL);
1174         }
1175         return (gp);
1176 }
1177
1178 static void
1179 g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1180     struct g_consumer *cp, struct g_provider *pp)
1181 {
1182         struct g_eli_softc *sc;
1183
1184         g_topology_assert();
1185         sc = gp->softc;
1186         if (sc == NULL)
1187                 return;
1188         if (pp != NULL || cp != NULL)
1189                 return; /* Nothing here. */
1190
1191         sbuf_printf(sb, "%s<KeysTotal>%ju</KeysTotal>\n", indent,
1192             (uintmax_t)sc->sc_ekeys_total);
1193         sbuf_printf(sb, "%s<KeysAllocated>%ju</KeysAllocated>\n", indent,
1194             (uintmax_t)sc->sc_ekeys_allocated);
1195         sbuf_printf(sb, "%s<Flags>", indent);
1196         if (sc->sc_flags == 0)
1197                 sbuf_printf(sb, "NONE");
1198         else {
1199                 int first = 1;
1200
1201 #define ADD_FLAG(flag, name)    do {                                    \
1202         if (sc->sc_flags & (flag)) {                                    \
1203                 if (!first)                                             \
1204                         sbuf_printf(sb, ", ");                          \
1205                 else                                                    \
1206                         first = 0;                                      \
1207                 sbuf_printf(sb, name);                                  \
1208         }                                                               \
1209 } while (0)
1210                 ADD_FLAG(G_ELI_FLAG_SUSPEND, "SUSPEND");
1211                 ADD_FLAG(G_ELI_FLAG_SINGLE_KEY, "SINGLE-KEY");
1212                 ADD_FLAG(G_ELI_FLAG_NATIVE_BYTE_ORDER, "NATIVE-BYTE-ORDER");
1213                 ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
1214                 ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
1215                 ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
1216                 ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
1217                 ADD_FLAG(G_ELI_FLAG_AUTH, "AUTH");
1218                 ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
1219                 ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
1220                 ADD_FLAG(G_ELI_FLAG_RO, "READ-ONLY");
1221                 ADD_FLAG(G_ELI_FLAG_NODELETE, "NODELETE");
1222                 ADD_FLAG(G_ELI_FLAG_GELIBOOT, "GELIBOOT");
1223 #undef  ADD_FLAG
1224         }
1225         sbuf_printf(sb, "</Flags>\n");
1226
1227         if (!(sc->sc_flags & G_ELI_FLAG_ONETIME)) {
1228                 sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
1229                     sc->sc_nkey);
1230         }
1231         sbuf_printf(sb, "%s<Version>%u</Version>\n", indent, sc->sc_version);
1232         sbuf_printf(sb, "%s<Crypto>", indent);
1233         switch (sc->sc_crypto) {
1234         case G_ELI_CRYPTO_HW:
1235                 sbuf_printf(sb, "hardware");
1236                 break;
1237         case G_ELI_CRYPTO_SW:
1238                 sbuf_printf(sb, "software");
1239                 break;
1240         default:
1241                 sbuf_printf(sb, "UNKNOWN");
1242                 break;
1243         }
1244         sbuf_printf(sb, "</Crypto>\n");
1245         if (sc->sc_flags & G_ELI_FLAG_AUTH) {
1246                 sbuf_printf(sb,
1247                     "%s<AuthenticationAlgorithm>%s</AuthenticationAlgorithm>\n",
1248                     indent, g_eli_algo2str(sc->sc_aalgo));
1249         }
1250         sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent,
1251             sc->sc_ekeylen);
1252         sbuf_printf(sb, "%s<EncryptionAlgorithm>%s</EncryptionAlgorithm>\n",
1253             indent, g_eli_algo2str(sc->sc_ealgo));
1254         sbuf_printf(sb, "%s<State>%s</State>\n", indent,
1255             (sc->sc_flags & G_ELI_FLAG_SUSPEND) ? "SUSPENDED" : "ACTIVE");
1256 }
1257
1258 static void
1259 g_eli_shutdown_pre_sync(void *arg, int howto)
1260 {
1261         struct g_class *mp;
1262         struct g_geom *gp, *gp2;
1263         struct g_provider *pp;
1264         struct g_eli_softc *sc;
1265         int error;
1266
1267         mp = arg;
1268         g_topology_lock();
1269         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
1270                 sc = gp->softc;
1271                 if (sc == NULL)
1272                         continue;
1273                 pp = LIST_FIRST(&gp->provider);
1274                 KASSERT(pp != NULL, ("No provider? gp=%p (%s)", gp, gp->name));
1275                 if (pp->acr + pp->acw + pp->ace == 0)
1276                         error = g_eli_destroy(sc, TRUE);
1277                 else {
1278                         sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
1279                         gp->access = g_eli_access;
1280                 }
1281         }
1282         g_topology_unlock();
1283 }
1284
1285 static void
1286 g_eli_init(struct g_class *mp)
1287 {
1288
1289         g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
1290             g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
1291         if (g_eli_pre_sync == NULL)
1292                 G_ELI_DEBUG(0, "Warning! Cannot register shutdown event.");
1293 }
1294
1295 static void
1296 g_eli_fini(struct g_class *mp)
1297 {
1298
1299         if (g_eli_pre_sync != NULL)
1300                 EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync);
1301 }
1302
1303 DECLARE_GEOM_CLASS(g_eli_class, g_eli);
1304 MODULE_DEPEND(g_eli, crypto, 1, 1, 1);