From fb5885af378f02ffb592a92b3811025642a47630 Mon Sep 17 00:00:00 2001 From: Lukas Ertl Date: Fri, 26 Nov 2004 12:01:00 +0000 Subject: [PATCH] Implement checkparity/rebuildparity. --- sbin/gvinum/gvinum.c | 88 ++++++++++++++++++++++ sys/geom/vinum/geom_vinum.c | 3 + sys/geom/vinum/geom_vinum.h | 1 + sys/geom/vinum/geom_vinum_init.c | 119 ++++++++++++++++++++++++++++++ sys/geom/vinum/geom_vinum_plex.c | 105 +++++++++++++++++++++++--- sys/geom/vinum/geom_vinum_raid5.c | 97 +++++++++++++++++++++++- sys/geom/vinum/geom_vinum_raid5.h | 2 + sys/geom/vinum/geom_vinum_var.h | 2 + 8 files changed, 404 insertions(+), 13 deletions(-) diff --git a/sbin/gvinum/gvinum.c b/sbin/gvinum/gvinum.c index 176c7c3d8aa..524a08ebbab 100644 --- a/sbin/gvinum/gvinum.c +++ b/sbin/gvinum/gvinum.c @@ -55,6 +55,7 @@ void gvinum_create(int, char **); void gvinum_help(void); void gvinum_init(int, char **); void gvinum_list(int, char **); +void gvinum_parityop(int, char **, int); void gvinum_printconfig(int, char **); void gvinum_rm(int, char **); void gvinum_saveconfig(void); @@ -551,6 +552,89 @@ gvinum_printconfig(int argc, char **argv) printconfig(stdout, ""); } +void +gvinum_parityop(int argc, char **argv, int rebuild) +{ + struct gctl_req *req; + int flags, i, rv; + off_t offset; + const char *errstr; + char *op, *msg; + + if (rebuild) { + op = "rebuildparity"; + msg = "Rebuilding"; + } else { + op = "checkparity"; + msg = "Checking"; + } + + optreset = 1; + optind = 1; + flags = 0; + while ((i = getopt(argc, argv, "fv")) != -1) { + switch (i) { + case 'f': + flags |= GV_FLAG_F; + break; + case 'v': + flags |= GV_FLAG_V; + break; + case '?': + default: + warnx("invalid flag '%c'", i); + return; + } + } + argc -= optind; + argv += optind; + + if (argc != 1) { + warn("usage: %s [-f] [-v] ", op); + return; + } + + do { + rv = 0; + req = gctl_get_handle(); + gctl_ro_param(req, "class", -1, "VINUM"); + gctl_ro_param(req, "verb", -1, "parityop"); + gctl_ro_param(req, "flags", sizeof(int), &flags); + gctl_ro_param(req, "rebuild", sizeof(int), &rebuild); + gctl_rw_param(req, "rv", sizeof(int), &rv); + gctl_rw_param(req, "offset", sizeof(off_t), &offset); + gctl_ro_param(req, "plex", -1, argv[0]); + errstr = gctl_issue(req); + if (errstr) { + warnx("%s\n", errstr); + gctl_free(req); + break; + } + gctl_free(req); + if (flags & GV_FLAG_V) { + printf("\r%s at %s ... ", msg, + gv_roughlength(offset, 1)); + } + if (rv == 1) { + printf("Parity incorrect at offset 0x%jx\n", + (intmax_t)offset); + if (!rebuild) + break; + } + fflush(stdout); + + /* Clear the -f flag. */ + flags &= ~GV_FLAG_F; + } while (rv >= 0); + + if ((rv == 2) && (flags & GV_FLAG_V)) { + if (rebuild) + printf("Rebuilt parity on %s\n", argv[0]); + else + printf("%s has correct parity\n", argv[0]); + } +} + void gvinum_rm(int argc, char **argv) { @@ -721,6 +805,10 @@ parseline(int argc, char **argv) gvinum_start(argc, argv); else if (!strcmp(argv[0], "stop")) gvinum_stop(argc, argv); + else if (!strcmp(argv[0], "checkparity")) + gvinum_parityop(argc, argv, 0); + else if (!strcmp(argv[0], "rebuildparity")) + gvinum_parityop(argc, argv, 1); else printf("unknown command '%s'\n", argv[0]); diff --git a/sys/geom/vinum/geom_vinum.c b/sys/geom/vinum/geom_vinum.c index a25e5cafd63..5a54bee5872 100644 --- a/sys/geom/vinum/geom_vinum.c +++ b/sys/geom/vinum/geom_vinum.c @@ -503,6 +503,9 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb) } else if (!strcmp(verb, "create")) { gv_create(gp, req); + } else if (!strcmp(verb, "parityop")) { + gv_parityop(gp, req); + } else if (!strcmp(verb, "remove")) { gv_remove(gp, req); diff --git a/sys/geom/vinum/geom_vinum.h b/sys/geom/vinum/geom_vinum.h index de48c7dbbe0..c215e2ea009 100644 --- a/sys/geom/vinum/geom_vinum.h +++ b/sys/geom/vinum/geom_vinum.h @@ -38,6 +38,7 @@ void gv_save_config(struct g_consumer *, struct gv_drive *, struct gv_softc *); /* geom_vinum_init.c */ +void gv_parityop(struct g_geom *, struct gctl_req *); void gv_start_obj(struct g_geom *, struct gctl_req *); /* geom_vinum_list.c */ diff --git a/sys/geom/vinum/geom_vinum_init.c b/sys/geom/vinum/geom_vinum_init.c index 46d9d51b969..95b3d6a8dfa 100644 --- a/sys/geom/vinum/geom_vinum_init.c +++ b/sys/geom/vinum/geom_vinum_init.c @@ -57,6 +57,125 @@ struct gv_sync_args { off_t syncsize; }; +void +gv_parityop(struct g_geom *gp, struct gctl_req *req) +{ + struct gv_softc *sc; + struct gv_plex *p; + struct bio *bp; + struct g_consumer *cp; + int error, *flags, type, *rebuild, rv; + char *plex; + + rv = -1; + + plex = gctl_get_param(req, "plex", NULL); + if (plex == NULL) { + gctl_error(req, "no plex given"); + goto out; + } + + flags = gctl_get_paraml(req, "flags", sizeof(*flags)); + if (flags == NULL) { + gctl_error(req, "no flags given"); + goto out; + } + + rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild)); + if (rebuild == NULL) { + gctl_error(req, "no rebuild op given"); + goto out; + } + + sc = gp->softc; + type = gv_object_type(sc, plex); + switch (type) { + case GV_TYPE_PLEX: + break; + case GV_TYPE_VOL: + case GV_TYPE_SD: + case GV_TYPE_DRIVE: + default: + gctl_error(req, "'%s' is not a plex", plex); + goto out; + } + + p = gv_find_plex(sc, plex); + if (p->state != GV_PLEX_UP) { + gctl_error(req, "plex %s is not completely accessible", + p->name); + goto out; + } + + cp = p->consumer; + error = g_access(cp, 1, 1, 0); + if (error) { + gctl_error(req, "cannot access consumer"); + goto out; + } + g_topology_unlock(); + + /* Reset the check pointer when using -f. */ + if (*flags & GV_FLAG_F) + p->synced = 0; + + bp = g_new_bio(); + if (bp == NULL) { + gctl_error(req, "cannot create BIO - out of memory"); + g_topology_lock(); + error = g_access(cp, -1, -1, 0); + goto out; + } + bp->bio_cmd = BIO_WRITE; + bp->bio_done = NULL; + bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO); + bp->bio_cflags |= GV_BIO_CHECK; + if (*rebuild) + bp->bio_cflags |= GV_BIO_PARITY; + bp->bio_offset = p->synced; + bp->bio_length = p->stripesize; + + /* Schedule it down ... */ + g_io_request(bp, cp); + + /* ... and wait for the result. */ + error = biowait(bp, "gwrite"); + g_free(bp->bio_data); + g_destroy_bio(bp); + + if (error) { + /* Incorrect parity. */ + if (error == EAGAIN) + rv = 1; + + /* Some other error happened. */ + else + gctl_error(req, "Parity check failed at offset 0x%jx, " + "errno %d", (intmax_t)p->synced, error); + + /* Correct parity. */ + } else + rv = 0; + + gctl_set_param(req, "offset", &p->synced, sizeof(p->synced)); + + /* Advance the checkpointer if there was no error. */ + if (rv == 0) + p->synced += p->stripesize; + + /* End of plex; reset the check pointer and signal it to the caller. */ + if (p->synced >= p->size) { + p->synced = 0; + rv = -2; + } + + g_topology_lock(); + error = g_access(cp, -1, -1, 0); + +out: + gctl_set_param(req, "rv", &rv, sizeof(rv)); +} + void gv_start_obj(struct g_geom *gp, struct gctl_req *req) { diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c index 31266f9ee3d..8fa29238d84 100644 --- a/sys/geom/vinum/geom_vinum_plex.c +++ b/sys/geom/vinum/geom_vinum_plex.c @@ -46,6 +46,10 @@ __FBSDID("$FreeBSD$"); static void gv_plex_completed_request(struct gv_plex *, struct bio *); static void gv_plex_normal_request(struct gv_plex *, struct bio *); static void gv_plex_worker(void *); +static int gv_check_parity(struct gv_plex *, struct bio *, + struct gv_raid5_packet *); +static int gv_normal_parity(struct gv_plex *, struct bio *, + struct gv_raid5_packet *); /* XXX: is this the place to catch dying subdisks? */ static void @@ -346,6 +350,85 @@ gv_plex_worker(void *arg) kthread_exit(ENXIO); } +static int +gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) +{ + struct bio *cbp, *pbp; + int finished, i; + + finished = 1; + + if (wp->waiting != NULL) { + pbp = wp->waiting; + wp->waiting = NULL; + cbp = wp->parity; + for (i = 0; i < wp->length; i++) + cbp->bio_data[i] ^= pbp->bio_data[i]; + g_io_request(pbp, pbp->bio_caller2); + finished = 0; + + } else if (wp->parity != NULL) { + cbp = wp->parity; + wp->parity = NULL; + g_io_request(cbp, cbp->bio_caller2); + finished = 0; + } + + return (finished); +} + +static int +gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) +{ + struct bio *cbp, *pbp; + int err, finished, i; + + err = 0; + finished = 1; + + if (wp->waiting != NULL) { + pbp = wp->waiting; + wp->waiting = NULL; + g_io_request(pbp, pbp->bio_caller2); + finished = 0; + + } else if (wp->parity != NULL) { + cbp = wp->parity; + wp->parity = NULL; + + /* Check if the parity is correct. */ + for (i = 0; i < wp->length; i++) { + if (bp->bio_data[i] != cbp->bio_data[i]) { + err = 1; + break; + } + } + + /* The parity is not correct... */ + if (err) { + bp->bio_parent->bio_error = EAGAIN; + + /* ... but we rebuild it. */ + if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) { + g_io_request(cbp, cbp->bio_caller2); + finished = 0; + } + } + + /* + * Clean up the BIO we would have used for rebuilding the + * parity. + */ + if (finished) { + bp->bio_parent->bio_inbed++; + g_destroy_bio(cbp); + } + + } + + return (finished); +} + void gv_plex_completed_request(struct gv_plex *p, struct bio *bp) { @@ -405,18 +488,13 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp) /* Handle parity data. */ if (TAILQ_EMPTY(&wp->bits)) { - if (wp->waiting != NULL) { - pbp = wp->waiting; - wp->waiting = NULL; - cbp = wp->parity; - for (i = 0; i < wp->length; i++) - cbp->bio_data[i] ^= pbp->bio_data[i]; - g_io_request(pbp, pbp->bio_caller2); - } else if (wp->parity != NULL) { - cbp = wp->parity; - wp->parity = NULL; - g_io_request(cbp, cbp->bio_caller2); - } else { + if (bp->bio_parent->bio_cflags & GV_BIO_CHECK) + i = gv_check_parity(p, bp, wp); + else + i = gv_normal_parity(p, bp, wp); + + /* All of our sub-requests have finished. */ + if (i) { bp->bio_parent->bio_completed += wp->length; TAILQ_REMOVE(&p->packets, wp, list); /* Bring the waiting bios back into the game. */ @@ -475,6 +553,9 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp) if (bp->bio_cflags & GV_BIO_REBUILD) err = gv_rebuild_raid5(p, wp, bp, addr, boff, bcount); + else if (bp->bio_cflags & GV_BIO_CHECK) + err = gv_check_raid5(p, wp, bp, addr, + boff, bcount); else err = gv_build_raid5_req(p, wp, bp, addr, boff, bcount); diff --git a/sys/geom/vinum/geom_vinum_raid5.c b/sys/geom/vinum/geom_vinum_raid5.c index 11bb9315494..789db193ecb 100644 --- a/sys/geom/vinum/geom_vinum_raid5.c +++ b/sys/geom/vinum/geom_vinum_raid5.c @@ -80,6 +80,101 @@ gv_stripe_active(struct gv_plex *p, struct bio *bp) return (overlap); } +int +gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp, + caddr_t addr, off_t boff, off_t bcount) +{ + struct gv_sd *parity, *s; + struct gv_bioq *bq; + struct bio *cbp, *pbp; + int i, psdno; + off_t real_len, real_off; + + if (p == NULL || LIST_EMPTY(&p->subdisks)) + return (ENXIO); + + gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, &psdno); + + /* Find the right subdisk. */ + parity = NULL; + i = 0; + LIST_FOREACH(s, &p->subdisks, in_plex) { + if (i == psdno) { + parity = s; + break; + } + i++; + } + + /* Parity stripe not found. */ + if (parity == NULL) + return (ENXIO); + + if (parity->state != GV_SD_UP) + return (ENXIO); + + wp->length = real_len; + wp->data = addr; + wp->lockbase = real_off; + + /* Read all subdisks. */ + LIST_FOREACH(s, &p->subdisks, in_plex) { + /* Skip the parity subdisk. */ + if (s == parity) + continue; + + cbp = g_clone_bio(bp); + if (cbp == NULL) + return (ENOMEM); + cbp->bio_cmd = BIO_READ; + cbp->bio_data = g_malloc(real_len, M_WAITOK); + cbp->bio_cflags |= GV_BIO_MALLOC; + cbp->bio_offset = real_off; + cbp->bio_length = real_len; + cbp->bio_done = gv_plex_done; + cbp->bio_caller2 = s->consumer; + cbp->bio_driver1 = wp; + + GV_ENQUEUE(bp, cbp, pbp); + + bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO); + bq->bp = cbp; + TAILQ_INSERT_TAIL(&wp->bits, bq, queue); + } + + /* Read the parity data. */ + cbp = g_clone_bio(bp); + if (cbp == NULL) + return (ENOMEM); + cbp->bio_cmd = BIO_READ; + cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO); + cbp->bio_cflags |= GV_BIO_MALLOC; + cbp->bio_offset = real_off; + cbp->bio_length = real_len; + cbp->bio_done = gv_plex_done; + cbp->bio_caller2 = parity->consumer; + cbp->bio_driver1 = wp; + wp->waiting = cbp; + + /* + * In case we want to rebuild the parity, create an extra BIO to write + * it out. It also acts as buffer for the XOR operations. + */ + cbp = g_clone_bio(bp); + if (cbp == NULL) + return (ENOMEM); + cbp->bio_data = addr; + cbp->bio_offset = real_off; + cbp->bio_length = real_len; + cbp->bio_done = gv_plex_done; + cbp->bio_caller2 = parity->consumer; + cbp->bio_driver1 = wp; + wp->parity = cbp; + + return (0); +} + +/* Rebuild a degraded RAID5 plex. */ int gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp, caddr_t addr, off_t boff, off_t bcount) @@ -101,7 +196,7 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp, broken = s; } - /* Parity stripe not found. */ + /* Broken stripe not found. */ if (broken == NULL) return (ENXIO); diff --git a/sys/geom/vinum/geom_vinum_raid5.h b/sys/geom/vinum/geom_vinum_raid5.h index 212f6c65f83..9637db1d5c4 100644 --- a/sys/geom/vinum/geom_vinum_raid5.h +++ b/sys/geom/vinum/geom_vinum_raid5.h @@ -67,6 +67,8 @@ struct gv_raid5_packet { int gv_stripe_active(struct gv_plex *, struct bio *); int gv_build_raid5_req(struct gv_plex *, struct gv_raid5_packet *, struct bio *, caddr_t, off_t, off_t); +int gv_check_raid5(struct gv_plex *, struct gv_raid5_packet *, + struct bio *, caddr_t, off_t, off_t); int gv_rebuild_raid5(struct gv_plex *, struct gv_raid5_packet *, struct bio *, caddr_t, off_t, off_t); void gv_raid5_worker(void *); diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h index 95d8956502f..05a5e8cad3c 100644 --- a/sys/geom/vinum/geom_vinum_var.h +++ b/sys/geom/vinum/geom_vinum_var.h @@ -114,6 +114,8 @@ #define GV_BIO_SYNCREQ 0x08 #define GV_BIO_SUCCEED 0x10 #define GV_BIO_REBUILD 0x20 +#define GV_BIO_CHECK 0x40 +#define GV_BIO_PARITY 0x80 /* * hostname is 256 bytes long, but we don't need to shlep multiple copies in -- 2.45.2