2 * Copyright (c) 2002 Poul-Henning Kamp
3 * Copyright (c) 2002 Networks Associates Technology, Inc.
6 * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7 * and NAI Labs, the Security Research Division of Network Associates, Inc.
8 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9 * DARPA CHATS research program.
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * This source file contains the state-engine which makes things happen in the
40 * Break the struct bio into multiple work packets one per zone.
42 * Setup the necessary sector buffers and start those read operations
43 * which we can start at this time and put the item on the work-list.
45 * Scan the work-list for items which are ready for crypto processing
46 * and call the matching crypto function in g_bde_crypt.c and schedule
47 * any writes needed. Read operations finish here by releasing the
48 * sector buffers and delivering the original bio request.
49 * 4) g_bde_write_done()
50 * Release sector buffers and deliver the original bio request.
52 * Because of the C-scope rules, the functions are almost perfectly in the
53 * opposite order in this source file.
55 * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add
56 * XXX: additional states to this state-engine. Since no hardware available
57 * XXX: at this time has AES support, implementing this has been postponed
58 * XXX: until such time as it would result in a benefit.
61 #include <sys/param.h>
64 #include <sys/mutex.h>
65 #include <sys/queue.h>
66 #include <sys/malloc.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/sysctl.h>
71 #include <sys/kthread.h>
73 #include <crypto/rijndael/rijndael-api-fst.h>
74 #include <crypto/sha2/sha2.h>
75 #include <geom/geom.h>
76 #include <geom/bde/g_bde.h>
78 static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp);
79 static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len);
80 static void g_bde_release_keysector(struct g_bde_work *wp);
81 static struct g_bde_sector *g_bde_get_keysector(struct g_bde_work *wp);
82 static int g_bde_start_read(struct g_bde_sector *sp);
83 static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction);
86 * Work item allocation.
88 * C++ would call these constructors and destructors.
90 static u_int g_bde_nwork;
91 SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, "");
93 static MALLOC_DEFINE(M_GBDE, "gbde", "GBDE data structures");
95 static struct g_bde_work *
96 g_bde_new_work(struct g_bde_softc *sc)
98 struct g_bde_work *wp;
100 wp = malloc(sizeof *wp, M_GBDE, M_NOWAIT | M_ZERO);
107 TAILQ_INSERT_TAIL(&sc->worklist, wp, list);
112 g_bde_delete_work(struct g_bde_work *wp)
114 struct g_bde_softc *sc;
119 TAILQ_REMOVE(&sc->worklist, wp, list);
124 * Sector buffer allocation
126 * These two functions allocate and free back variable sized sector buffers
129 static u_int g_bde_nsect;
130 SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, "");
133 g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
139 free(sp->data, M_GBDE);
143 static struct g_bde_sector *
144 g_bde_new_sector(struct g_bde_work *wp, u_int len)
146 struct g_bde_sector *sp;
148 sp = malloc(sizeof *sp, M_GBDE, M_NOWAIT | M_ZERO);
152 sp->data = malloc(len, M_GBDE, M_NOWAIT | M_ZERO);
153 if (sp->data == NULL) {
162 sp->softc = wp->softc;
173 * Nothing prevents two separate I/O requests from addressing the same zone
174 * and thereby needing the same skey sector. We therefore need to sequence
175 * I/O operations to the skey sectors. A certain amount of caching is also
176 * desirable, although the extent of benefit from this is not at this point
179 * XXX: GEOM may be able to grow a generic caching facility at some point
180 * XXX: to support such needs.
183 static u_int g_bde_ncache;
184 SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, "");
187 g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp)
190 g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp);
193 TAILQ_REMOVE(&sc->freelist, sp, list);
196 bzero(sp->data, sp->size);
197 g_bde_delete_sector(sc, sp);
200 static struct g_bde_sector *
201 g_bde_get_keysector(struct g_bde_work *wp)
203 struct g_bde_sector *sp;
204 struct g_bde_softc *sc;
208 g_trace(G_T_TOPOLOGY, "g_bde_get_keysector(%p, %jd)", wp, (intmax_t)offset);
211 if (malloc_last_fail() < g_bde_ncache)
212 g_bde_purge_sector(sc, -1);
214 sp = TAILQ_FIRST(&sc->freelist);
215 if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime)
216 g_bde_purge_one_sector(sc, sp);
218 TAILQ_FOREACH(sp, &sc->freelist, list) {
219 if (sp->offset == offset)
224 KASSERT(sp->offset == offset, ("wrong offset"));
225 KASSERT(sp->softc == wp->softc, ("wrong softc"));
229 if (malloc_last_fail() < g_bde_ncache) {
230 TAILQ_FOREACH(sp, &sc->freelist, list)
234 if (sp == NULL && !TAILQ_EMPTY(&sc->freelist))
235 sp = TAILQ_FIRST(&sc->freelist);
236 if (sp != NULL && sp->ref > 0)
239 sp = g_bde_new_sector(wp, sc->sectorsize);
243 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
249 sp->softc = wp->softc;
257 TAILQ_REMOVE(&sc->freelist, sp, list);
258 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
259 sp->used = time_uptime;
266 g_bde_release_keysector(struct g_bde_work *wp)
268 struct g_bde_softc *sc;
269 struct g_bde_work *wp2;
270 struct g_bde_sector *sp;
273 g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp);
274 KASSERT(sp->malloc == 2, ("Wrong sector released"));
276 KASSERT(sc != NULL, ("NULL sp->softc"));
277 KASSERT(wp == sp->owner, ("Releasing, not owner"));
282 TAILQ_REMOVE(&sc->freelist, sp, list);
283 TAILQ_INSERT_TAIL(&sc->freelist, sp, list);
284 TAILQ_FOREACH(wp2, &sc->worklist, list) {
285 if (wp2->ksp == sp) {
286 KASSERT(wp2 != wp, ("Self-reowning"));
292 KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp));
293 } else if (sp->error != 0) {
298 TAILQ_REMOVE(&sc->freelist, sp, list);
299 TAILQ_INSERT_HEAD(&sc->freelist, sp, list);
303 g_bde_purge_sector(struct g_bde_softc *sc, int fraction)
305 struct g_bde_sector *sp;
308 g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc);
310 n = sc->ncache / fraction + 1;
312 n = g_bde_ncache - malloc_last_fail();
318 TAILQ_FOREACH(sp, &sc->freelist, list) {
321 TAILQ_REMOVE(&sc->freelist, sp, list);
324 bzero(sp->data, sp->size);
325 g_bde_delete_sector(sc, sp);
331 static struct g_bde_sector *
332 g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp)
334 struct g_bde_sector *sp;
336 g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp);
337 sp = g_bde_get_keysector(wp);
339 g_bde_purge_sector(sc, -1);
340 sp = g_bde_get_keysector(wp);
346 if (sp->state == VALID)
348 if (g_bde_start_read(sp) == 0)
350 g_bde_release_keysector(wp);
355 * Contribute to the completion of the original bio request.
357 * We have no simple way to tell how many bits the original bio request has
358 * been segmented into, so the easiest way to determine when we can deliver
359 * it is to keep track of the number of bytes we have completed. We keep
360 * track of any errors underway and latch onto the first one.
362 * We always report "nothing done" in case of error, because random bits here
363 * and there may be completed and returning a number of completed bytes does
364 * not convey any useful information about which bytes they were. If some
365 * piece of broken code somewhere interprets this to mean that nothing has
366 * changed on the underlying media they deserve the lossage headed for them.
368 * A single mutex per g_bde instance is used to prevent contention.
372 g_bde_contribute(struct bio *bp, off_t bytes, int error)
375 g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d",
376 bp, (intmax_t)bytes, error);
377 if (bp->bio_error == 0)
378 bp->bio_error = error;
379 bp->bio_completed += bytes;
380 KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution"));
381 if (bp->bio_completed == bp->bio_length) {
382 if (bp->bio_error != 0)
383 bp->bio_completed = 0;
384 g_io_deliver(bp, bp->bio_error);
389 * This is the common case "we're done with this work package" function
393 g_bde_work_done(struct g_bde_work *wp, int error)
396 g_bde_contribute(wp->bp, wp->length, error);
398 g_bde_delete_sector(wp->softc, wp->sp);
400 g_bde_release_keysector(wp);
401 g_bde_delete_work(wp);
405 * A write operation has finished. When we have all expected cows in the
406 * barn close the door and call it a day.
410 g_bde_write_done(struct bio *bp)
412 struct g_bde_sector *sp;
413 struct g_bde_work *wp;
414 struct g_bde_softc *sc;
416 sp = bp->bio_caller1;
417 sc = bp->bio_caller2;
418 mtx_lock(&sc->worklist_mutex);
419 KASSERT(sp != NULL, ("NULL sp"));
420 KASSERT(sc != NULL, ("NULL sc"));
421 KASSERT(sp->owner != NULL, ("NULL sp->owner"));
422 g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp);
423 if (bp->bio_error == 0 && bp->bio_completed != sp->size)
425 sp->error = bp->bio_error;
429 wp->error = sp->error;
431 if (wp->bp->bio_cmd == BIO_DELETE) {
432 KASSERT(sp == wp->sp, ("trashed delete op"));
433 g_bde_work_done(wp, wp->error);
434 mtx_unlock(&sc->worklist_mutex);
438 KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()"));
439 KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op"));
441 g_bde_delete_sector(sc, wp->sp);
446 if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID)
447 g_bde_work_done(wp, wp->error);
448 mtx_unlock(&sc->worklist_mutex);
453 * Send a write request for the given sector down the pipeline.
457 g_bde_start_write(struct g_bde_sector *sp)
460 struct g_bde_softc *sc;
462 g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp);
464 KASSERT(sc != NULL, ("NULL sc in g_bde_start_write"));
465 KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write"));
469 bp->bio_cmd = BIO_WRITE;
470 bp->bio_offset = sp->offset;
471 bp->bio_data = sp->data;
472 bp->bio_length = sp->size;
473 bp->bio_done = g_bde_write_done;
474 bp->bio_caller1 = sp;
475 bp->bio_caller2 = sc;
477 g_io_request(bp, sc->consumer);
482 * A read operation has finished. Mark the sector no longer iobusy and
483 * wake up the worker thread and let it do its thing.
487 g_bde_read_done(struct bio *bp)
489 struct g_bde_sector *sp;
490 struct g_bde_softc *sc;
492 sp = bp->bio_caller1;
493 g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp);
494 sc = bp->bio_caller2;
495 mtx_lock(&sc->worklist_mutex);
496 if (bp->bio_error == 0 && bp->bio_completed != sp->size)
498 sp->error = bp->bio_error;
505 mtx_unlock(&sc->worklist_mutex);
509 * Send a read request for the given sector down the pipeline.
513 g_bde_start_read(struct g_bde_sector *sp)
516 struct g_bde_softc *sc;
518 g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp);
520 KASSERT(sc != NULL, ("Null softc in sp %p", sp));
524 bp->bio_cmd = BIO_READ;
525 bp->bio_offset = sp->offset;
526 bp->bio_data = sp->data;
527 bp->bio_length = sp->size;
528 bp->bio_done = g_bde_read_done;
529 bp->bio_caller1 = sp;
530 bp->bio_caller2 = sc;
532 g_io_request(bp, sc->consumer);
539 * The up/down path of GEOM is not allowed to sleep or do any major work
540 * so we use this thread to do the actual crypto operations and to push
541 * the state engine onwards.
543 * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption
544 * XXX: using a thread here is probably not needed.
548 g_bde_worker(void *arg)
550 struct g_bde_softc *sc;
551 struct g_bde_work *wp, *twp;
558 mtx_lock(&sc->worklist_mutex);
561 g_trace(G_T_TOPOLOGY, "g_bde_worker scan");
562 TAILQ_FOREACH_SAFE(wp, &sc->worklist, list, twp) {
563 KASSERT(wp != NULL, ("NULL wp"));
564 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
565 if (wp->state != WAIT)
566 continue; /* Not interesting here */
568 KASSERT(wp->bp != NULL, ("NULL wp->bp"));
569 KASSERT(wp->sp != NULL, ("NULL wp->sp"));
571 if (wp->ksp != NULL) {
572 if (wp->ksp->owner != wp)
574 if (wp->ksp->state == IO)
576 KASSERT(wp->ksp->state == VALID,
577 ("Illegal sector state (%d)",
581 if (wp->bp->bio_cmd == BIO_READ && wp->sp->state == IO)
584 if (wp->ksp != NULL && wp->ksp->error != 0) {
585 g_bde_work_done(wp, wp->ksp->error);
588 switch(wp->bp->bio_cmd) {
590 if (wp->ksp == NULL) {
591 KASSERT(wp->error != 0,
592 ("BIO_READ, no ksp and no error"));
593 g_bde_work_done(wp, wp->error);
596 if (wp->sp->error != 0) {
597 g_bde_work_done(wp, wp->sp->error);
600 mtx_unlock(&sc->worklist_mutex);
601 g_bde_crypt_read(wp);
602 mtx_lock(&sc->worklist_mutex);
604 g_bde_work_done(wp, wp->sp->error);
608 KASSERT(wp->sp->owner == wp,
609 ("Write not owner sp"));
610 KASSERT(wp->ksp->owner == wp,
611 ("Write not owner ksp"));
612 mtx_unlock(&sc->worklist_mutex);
613 g_bde_crypt_write(wp);
614 mtx_lock(&sc->worklist_mutex);
616 error = g_bde_start_write(wp->sp);
618 g_bde_work_done(wp, error);
621 error = g_bde_start_write(wp->ksp);
627 mtx_unlock(&sc->worklist_mutex);
628 g_bde_crypt_delete(wp);
629 mtx_lock(&sc->worklist_mutex);
631 g_bde_start_write(wp->sp);
639 * We don't look for our death-warrant until we are
640 * idle. Shouldn't make a difference in practice.
644 g_trace(G_T_TOPOLOGY, "g_bde_worker sleep");
645 error = msleep(sc, &sc->worklist_mutex,
647 if (error == EWOULDBLOCK) {
649 * Lose our skey cache in an orderly fashion.
650 * The exact rate can be tuned to be less
651 * aggressive if this is desirable. 10% per
652 * second means that the cache is gone in a
655 g_bde_purge_sector(sc, 10);
659 g_trace(G_T_TOPOLOGY, "g_bde_worker die");
660 g_bde_purge_sector(sc, 1);
661 KASSERT(sc->nwork == 0, ("Dead but %d work remaining", sc->nwork));
662 KASSERT(sc->ncache == 0, ("Dead but %d cache remaining", sc->ncache));
663 KASSERT(sc->nsect == 0, ("Dead but %d sect remaining", sc->nsect));
664 mtx_unlock(&sc->worklist_mutex);
671 * g_bde_start1 has chopped the incoming request up so all the requests
672 * we see here are inside a single zone. Map the data and key locations
673 * grab the buffers we need and fire off the first volley of read requests.
677 g_bde_start2(struct g_bde_work *wp)
679 struct g_bde_softc *sc;
681 KASSERT(wp != NULL, ("NULL wp in g_bde_start2"));
682 KASSERT(wp->softc != NULL, ("NULL wp->softc"));
683 g_trace(G_T_TOPOLOGY, "g_bde_start2(%p)", wp);
685 switch (wp->bp->bio_cmd) {
687 wp->sp = g_bde_new_sector(wp, 0);
688 if (wp->sp == NULL) {
689 g_bde_work_done(wp, ENOMEM);
692 wp->sp->size = wp->length;
693 wp->sp->data = wp->data;
694 if (g_bde_start_read(wp->sp) != 0) {
695 g_bde_work_done(wp, ENOMEM);
698 g_bde_read_keysector(sc, wp);
703 wp->sp = g_bde_new_sector(wp, wp->length);
704 if (wp->sp == NULL) {
705 g_bde_work_done(wp, ENOMEM);
710 wp->sp = g_bde_new_sector(wp, wp->length);
711 if (wp->sp == NULL) {
712 g_bde_work_done(wp, ENOMEM);
715 g_bde_read_keysector(sc, wp);
716 if (wp->ksp == NULL) {
717 g_bde_work_done(wp, ENOMEM);
723 ("Wrong bio_cmd %d in g_bde_start2", wp->bp->bio_cmd));
731 * Create a sequence of work structures, and have g_bde_map_sector() determine
732 * how long they each can be. Feed them to g_bde_start2().
736 g_bde_start1(struct bio *bp)
738 struct g_bde_softc *sc;
739 struct g_bde_work *wp;
742 sc = bp->bio_to->geom->softc;
743 bp->bio_driver1 = sc;
745 mtx_lock(&sc->worklist_mutex);
746 for(done = 0; done < bp->bio_length; ) {
747 wp = g_bde_new_work(sc);
750 wp->offset = bp->bio_offset + done;
751 wp->data = bp->bio_data + done;
752 wp->length = bp->bio_length - done;
753 g_bde_map_sector(wp);
757 if (wp == NULL || bp->bio_error != 0) {
758 g_bde_contribute(bp, bp->bio_length - done, ENOMEM);
762 mtx_unlock(&sc->worklist_mutex);