From 7d874f0f36aaa894b5bb85336b7e7f955b208d30 Mon Sep 17 00:00:00 2001 From: Alan Somers Date: Wed, 26 Aug 2020 02:44:35 +0000 Subject: [PATCH] geli: use unmapped I/O Use unmapped I/O for geli. Unlike most geom providers, geli needs to manipulate data on every read or write. Previously it would always map bios. On my 16-core, dual socket server using geli atop md(4) devices, with 512B sectors, this change increases geli IOPs by about 3x. Note that geli still can't use unmapped I/O when data integrity verification is enabled (but it could, with a little more work). And it can't use unmapped I/O in combination with ZFS, because ZFS uses mapped bios. Reviewed by: markj, kib, jhb, mjg, mat, bcr (manpages) MFC after: 1 week Sponsored by: Axcient Differential Revision: https://reviews.freebsd.org/D25671 --- sys/geom/eli/g_eli.c | 11 ++++++ sys/geom/eli/g_eli_privacy.c | 65 +++++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index b246b83c1d7..842fce32472 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -49,6 +49,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include + #include #include @@ -972,6 +974,15 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp, */ pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX); pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; + if (CRYPTO_HAS_VMPAGE) { + /* + * On DMAP architectures we can use unmapped I/O. But don't + * use it with data integrity verification. That code hasn't + * been written yet. + */ + if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0) + pp->flags |= G_PF_ACCEPT_UNMAPPED; + } pp->mediasize = sc->sc_mediasize; pp->sectorsize = sc->sc_sectorsize; LIST_FOREACH(gap, &bpp->aliases, ga_next) diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index 7bd8eb29cc2..adb353441e3 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -62,6 +62,28 @@ __FBSDID("$FreeBSD$"); MALLOC_DECLARE(M_ELI); +/* + * Copy data from a (potentially unmapped) bio to a kernelspace buffer. + * + * The buffer must have at least as much room as bp->bio_length. + */ +static void +g_eli_bio_copyin(struct bio *bp, void *kaddr) +{ + struct uio uio; + struct iovec iov[1]; + + iov[0].iov_base = kaddr; + iov[0].iov_len = bp->bio_length; + uio.uio_iov = iov; + uio.uio_iovcnt = 1; + uio.uio_offset = 0; + uio.uio_resid = bp->bio_length; + uio.uio_segflg = UIO_SYSSPACE; + uio.uio_rw = UIO_READ; + uiomove_fromphys(bp->bio_ma, bp->bio_ma_offset, bp->bio_length, &uio); +} + /* * The function is called after we read and decrypt data. * @@ -98,8 +120,7 @@ g_eli_crypto_read_done(struct cryptop *crp) */ if (bp->bio_inbed < bp->bio_children) return (0); - free(bp->bio_driver2, M_ELI); - bp->bio_driver2 = NULL; + if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", bp->bio_error); @@ -167,6 +188,11 @@ g_eli_crypto_write_done(struct cryptop *crp) return (0); } cbp->bio_data = bp->bio_driver2; + /* + * Clear BIO_UNMAPPED, which was inherited from where we cloned the bio + * in g_eli_start, because we manually set bio_data + */ + cbp->bio_flags &= ~BIO_UNMAPPED; cbp->bio_done = g_eli_write_done; cp = LIST_FIRST(&gp->consumer); cbp->bio_to = cp->provider; @@ -236,10 +262,12 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) { struct g_eli_softc *sc; struct cryptop *crp; + vm_page_t *pages; u_int i, nsec, secsize; off_t dstoff; - u_char *data; + u_char *data = NULL; int error; + int pages_offset; G_ELI_LOGREQ(3, bp, "%s", __func__); @@ -258,16 +286,37 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) if (bp->bio_cmd == BIO_WRITE) { data = malloc(bp->bio_length, M_ELI, M_WAITOK); bp->bio_driver2 = data; - bcopy(bp->bio_data, data, bp->bio_length); - } else - data = bp->bio_data; + /* + * This copy could be eliminated by using crypto's output + * buffer, instead of using a single overwriting buffer. + */ + if ((bp->bio_flags & BIO_UNMAPPED) != 0) + g_eli_bio_copyin(bp, data); + else + bcopy(bp->bio_data, data, bp->bio_length); + } else { + if ((bp->bio_flags & BIO_UNMAPPED) != 0) { + pages = bp->bio_ma; + pages_offset = bp->bio_ma_offset; + } else { + data = bp->bio_data; + } + } for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) { crp = crypto_getreq(wr->w_sid, M_WAITOK); - crypto_use_buf(crp, data, secsize); + if (data) { + crypto_use_buf(crp, data, secsize); + data += secsize; + } else { + MPASS(pages != NULL); + crypto_use_vmpage(crp, pages, secsize, pages_offset); + pages_offset += secsize; + pages += pages_offset >> PAGE_SHIFT; + pages_offset &= PAGE_MASK; + } crp->crp_opaque = (void *)bp; - data += secsize; if (bp->bio_cmd == BIO_WRITE) { crp->crp_op = CRYPTO_OP_ENCRYPT; crp->crp_callback = g_eli_crypto_write_done; -- 2.45.0