sys/geom/vinum/geom_vinum.c

   1 /*-
   2  *  Copyright (c) 2004, 2007 Lukas Ertl
   3  *  Copyright (c) 2007, 2009 Ulf Lilleengen
   4  *  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/bio.h>
  34 #include <sys/kernel.h>
  35 #include <sys/kthread.h>
  36 #include <sys/lock.h>
  37 #include <sys/malloc.h>
  38 #include <sys/module.h>
  39 #include <sys/mutex.h>
  40 #include <sys/sbuf.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/systm.h>
  43
  44 #include <geom/geom.h>
  45 #include <geom/vinum/geom_vinum_var.h>
  46 #include <geom/vinum/geom_vinum.h>
  47 #include <geom/vinum/geom_vinum_raid5.h>
  48
  49 SYSCTL_DECL(_kern_geom);
  50 SYSCTL_NODE(_kern_geom, OID_AUTO, vinum, CTLFLAG_RW, 0, "GEOM_VINUM stuff");
  51 u_int g_vinum_debug = 0;
  52 TUNABLE_INT("kern.geom.vinum.debug", &g_vinum_debug);
  53 SYSCTL_UINT(_kern_geom_vinum, OID_AUTO, debug, CTLFLAG_RW, &g_vinum_debug, 0,
  54     "Debug level");
  55
  56 static int      gv_create(struct g_geom *, struct gctl_req *);
  57 static void     gv_attach(struct gv_softc *, struct gctl_req *);
  58 static void     gv_detach(struct gv_softc *, struct gctl_req *);
  59 static void     gv_parityop(struct gv_softc *, struct gctl_req *);
  60
  61
  62 static void
  63 gv_orphan(struct g_consumer *cp)
  64 {
  65         struct g_geom *gp;
  66         struct gv_softc *sc;
  67         struct gv_drive *d;
  68
  69         g_topology_assert();
  70
  71         KASSERT(cp != NULL, ("gv_orphan: null cp"));
  72         gp = cp->geom;
  73         KASSERT(gp != NULL, ("gv_orphan: null gp"));
  74         sc = gp->softc;
  75         KASSERT(sc != NULL, ("gv_orphan: null sc"));
  76         d = cp->private;
  77         KASSERT(d != NULL, ("gv_orphan: null d"));
  78
  79         g_trace(G_T_TOPOLOGY, "gv_orphan(%s)", gp->name);
  80
  81         gv_post_event(sc, GV_EVENT_DRIVE_LOST, d, NULL, 0, 0);
  82 }
  83
  84 void
  85 gv_start(struct bio *bp)
  86 {
  87         struct g_geom *gp;
  88         struct gv_softc *sc;
  89
  90         gp = bp->bio_to->geom;
  91         sc = gp->softc;
  92
  93         switch (bp->bio_cmd) {
  94         case BIO_READ:
  95         case BIO_WRITE:
  96         case BIO_DELETE:
  97                 break;
  98         case BIO_GETATTR:
  99         default:
 100                 g_io_deliver(bp, EOPNOTSUPP);
 101                 return;
 102         }
 103         mtx_lock(&sc->bqueue_mtx);
 104         bioq_disksort(sc->bqueue_down, bp);
 105         wakeup(sc);
 106         mtx_unlock(&sc->bqueue_mtx);
 107 }
 108
 109 void
 110 gv_done(struct bio *bp)
 111 {
 112         struct g_geom *gp;
 113         struct gv_softc *sc;
 114
 115         KASSERT(bp != NULL, ("NULL bp"));
 116
 117         gp = bp->bio_from->geom;
 118         sc = gp->softc;
 119
 120         mtx_lock(&sc->bqueue_mtx);
 121         bioq_disksort(sc->bqueue_up, bp);
 122         wakeup(sc);
 123         mtx_unlock(&sc->bqueue_mtx);
 124 }
 125
 126 int
 127 gv_access(struct g_provider *pp, int dr, int dw, int de)
 128 {
 129         struct g_geom *gp;
 130         struct gv_softc *sc;
 131         struct gv_drive *d, *d2;
 132         int error;
 133
 134         gp = pp->geom;
 135         sc = gp->softc;
 136         /*
 137          * We want to modify the read count with the write count in case we have
 138          * plexes in a RAID-5 organization.
 139          */
 140         dr += dw;
 141
 142         LIST_FOREACH(d, &sc->drives, drive) {
 143                 if (d->consumer == NULL)
 144                         continue;
 145                 error = g_access(d->consumer, dr, dw, de);
 146                 if (error) {
 147                         LIST_FOREACH(d2, &sc->drives, drive) {
 148                                 if (d == d2)
 149                                         break;
 150                                 g_access(d2->consumer, -dr, -dw, -de);
 151                         }
 152                         G_VINUM_DEBUG(0, "g_access '%s' failed: %d", d->name,
 153                             error);
 154                         return (error);
 155                 }
 156         }
 157         return (0);
 158 }
 159
 160 static void
 161 gv_init(struct g_class *mp)
 162 {
 163         struct g_geom *gp;
 164         struct gv_softc *sc;
 165
 166         g_trace(G_T_TOPOLOGY, "gv_init(%p)", mp);
 167
 168         gp = g_new_geomf(mp, "VINUM");
 169         gp->spoiled = gv_orphan;
 170         gp->orphan = gv_orphan;
 171         gp->access = gv_access;
 172         gp->start = gv_start;
 173         gp->softc = g_malloc(sizeof(struct gv_softc), M_WAITOK | M_ZERO);
 174         sc = gp->softc;
 175         sc->geom = gp;
 176         sc->bqueue_down = g_malloc(sizeof(struct bio_queue_head),
 177             M_WAITOK | M_ZERO);
 178         sc->bqueue_up = g_malloc(sizeof(struct bio_queue_head),
 179             M_WAITOK | M_ZERO);
 180         bioq_init(sc->bqueue_down);
 181         bioq_init(sc->bqueue_up);
 182         LIST_INIT(&sc->drives);
 183         LIST_INIT(&sc->subdisks);
 184         LIST_INIT(&sc->plexes);
 185         LIST_INIT(&sc->volumes);
 186         TAILQ_INIT(&sc->equeue);
 187         mtx_init(&sc->config_mtx, "gv_config", NULL, MTX_DEF);
 188         mtx_init(&sc->equeue_mtx, "gv_equeue", NULL, MTX_DEF);
 189         mtx_init(&sc->bqueue_mtx, "gv_bqueue", NULL, MTX_DEF);
 190         kproc_create(gv_worker, sc, &sc->worker, 0, 0, "gv_worker");
 191 }
 192
 193 static int
 194 gv_unload(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
 195 {
 196         struct gv_softc *sc;
 197
 198         g_trace(G_T_TOPOLOGY, "gv_unload(%p)", mp);
 199
 200         g_topology_assert();
 201         sc = gp->softc;
 202
 203         if (sc != NULL) {
 204                 gv_worker_exit(sc);
 205                 gp->softc = NULL;
 206                 g_wither_geom(gp, ENXIO);
 207         }
 208
 209         return (0);
 210 }
 211
 212 /* Handle userland request of attaching object. */
 213 static void
 214 gv_attach(struct gv_softc *sc, struct gctl_req *req)
 215 {
 216         struct gv_volume *v;
 217         struct gv_plex *p;
 218         struct gv_sd *s;
 219         off_t *offset;
 220         int *rename, type_child, type_parent;
 221         char *child, *parent;
 222
 223         child = gctl_get_param(req, "child", NULL);
 224         if (child == NULL) {
 225                 gctl_error(req, "no child given");
 226                 return;
 227         }
 228         parent = gctl_get_param(req, "parent", NULL);
 229         if (parent == NULL) {
 230                 gctl_error(req, "no parent given");
 231                 return;
 232         }
 233         offset = gctl_get_paraml(req, "offset", sizeof(*offset));
 234         if (offset == NULL) {
 235                 gctl_error(req, "no offset given");
 236                 return;
 237         }
 238         rename = gctl_get_paraml(req, "rename", sizeof(*rename));
 239         if (rename == NULL) {
 240                 gctl_error(req, "no rename flag given");
 241                 return;
 242         }
 243
 244         type_child = gv_object_type(sc, child);
 245         type_parent = gv_object_type(sc, parent);
 246
 247         switch (type_child) {
 248         case GV_TYPE_PLEX:
 249                 if (type_parent != GV_TYPE_VOL) {
 250                         gctl_error(req, "no such volume to attach to");
 251                         return;
 252                 }
 253                 v = gv_find_vol(sc, parent);
 254                 p = gv_find_plex(sc, child);
 255                 gv_post_event(sc, GV_EVENT_ATTACH_PLEX, p, v, *offset, *rename);
 256                 break;
 257         case GV_TYPE_SD:
 258                 if (type_parent != GV_TYPE_PLEX) {
 259                         gctl_error(req, "no such plex to attach to");
 260                         return;
 261                 }
 262                 p = gv_find_plex(sc, parent);
 263                 s = gv_find_sd(sc, child);
 264                 gv_post_event(sc, GV_EVENT_ATTACH_SD, s, p, *offset, *rename);
 265                 break;
 266         default:
 267                 gctl_error(req, "invalid child type");
 268                 break;
 269         }
 270 }
 271
 272 /* Handle userland request of detaching object. */
 273 static void
 274 gv_detach(struct gv_softc *sc, struct gctl_req *req)
 275 {
 276         struct gv_plex *p;
 277         struct gv_sd *s;
 278         int *flags, type;
 279         char *object;
 280
 281         object = gctl_get_param(req, "object", NULL);
 282         if (object == NULL) {
 283                 gctl_error(req, "no argument given");
 284                 return;
 285         }
 286
 287         flags = gctl_get_paraml(req, "flags", sizeof(*flags));
 288         type = gv_object_type(sc, object);
 289         switch (type) {
 290         case GV_TYPE_PLEX:
 291                 p = gv_find_plex(sc, object);
 292                 gv_post_event(sc, GV_EVENT_DETACH_PLEX, p, NULL, *flags, 0);
 293                 break;
 294         case GV_TYPE_SD:
 295                 s = gv_find_sd(sc, object);
 296                 gv_post_event(sc, GV_EVENT_DETACH_SD, s, NULL, *flags, 0);
 297                 break;
 298         default:
 299                 gctl_error(req, "invalid object type");
 300                 break;
 301         }
 302 }
 303
 304 /* Handle userland requests for creating new objects. */
 305 static int
 306 gv_create(struct g_geom *gp, struct gctl_req *req)
 307 {
 308         struct gv_softc *sc;
 309         struct gv_drive *d, *d2;
 310         struct gv_plex *p, *p2;
 311         struct gv_sd *s, *s2;
 312         struct gv_volume *v, *v2;
 313         struct g_provider *pp;
 314         int error, i, *drives, *flags, *plexes, *subdisks, *volumes;
 315         char buf[20];
 316
 317         g_topology_assert();
 318
 319         sc = gp->softc;
 320
 321         /* Find out how many of each object have been passed in. */
 322         volumes = gctl_get_paraml(req, "volumes", sizeof(*volumes));
 323         plexes = gctl_get_paraml(req, "plexes", sizeof(*plexes));
 324         subdisks = gctl_get_paraml(req, "subdisks", sizeof(*subdisks));
 325         drives = gctl_get_paraml(req, "drives", sizeof(*drives));
 326         if (volumes == NULL || plexes == NULL || subdisks == NULL ||
 327             drives == NULL) {
 328                 gctl_error(req, "number of objects not given");
 329                 return (-1);
 330         }
 331         flags = gctl_get_paraml(req, "flags", sizeof(*flags));
 332         if (flags == NULL) {
 333                 gctl_error(req, "flags not given");
 334                 return (-1);
 335         }
 336
 337         /* First, handle drive definitions ... */
 338         for (i = 0; i < *drives; i++) {
 339                 snprintf(buf, sizeof(buf), "drive%d", i);
 340                 d2 = gctl_get_paraml(req, buf, sizeof(*d2));
 341                 if (d2 == NULL) {
 342                         gctl_error(req, "no drive definition given");
 343                         return (-1);
 344                 }
 345                 /*
 346                  * Make sure that the device specified in the drive config is
 347                  * an active GEOM provider.
 348                  */
 349                 pp = g_provider_by_name(d2->device);
 350                 if (pp == NULL) {
 351                         gctl_error(req, "%s: device not found", d2->device);
 352                         goto error;
 353                 }
 354                 if (gv_find_drive(sc, d2->name) != NULL) {
 355                         /* Ignore error. */
 356                         if (*flags & GV_FLAG_F)
 357                                 continue;
 358                         gctl_error(req, "drive '%s' already exists", d2->name);
 359                         goto error;
 360                 }
 361                 if (gv_find_drive_device(sc, d2->device) != NULL) {
 362                         gctl_error(req, "device '%s' already configured in "
 363                             "gvinum", d2->device);
 364                         goto error;
 365                 }
 366
 367
 368                 d = g_malloc(sizeof(*d), M_WAITOK | M_ZERO);
 369                 bcopy(d2, d, sizeof(*d));
 370
 371                 gv_post_event(sc, GV_EVENT_CREATE_DRIVE, d, NULL, 0, 0);
 372         }
 373
 374         /* ... then volume definitions ... */
 375         for (i = 0; i < *volumes; i++) {
 376                 error = 0;
 377                 snprintf(buf, sizeof(buf), "volume%d", i);
 378                 v2 = gctl_get_paraml(req, buf, sizeof(*v2));
 379                 if (v2 == NULL) {
 380                         gctl_error(req, "no volume definition given");
 381                         return (-1);
 382                 }
 383                 if (gv_find_vol(sc, v2->name) != NULL) {
 384                         /* Ignore error. */
 385                         if (*flags & GV_FLAG_F)
 386                                 continue;
 387                         gctl_error(req, "volume '%s' already exists", v2->name);
 388                         goto error;
 389                 }
 390
 391                 v = g_malloc(sizeof(*v), M_WAITOK | M_ZERO);
 392                 bcopy(v2, v, sizeof(*v));
 393
 394                 gv_post_event(sc, GV_EVENT_CREATE_VOLUME, v, NULL, 0, 0);
 395         }
 396
 397         /* ... then plex definitions ... */
 398         for (i = 0; i < *plexes; i++) {
 399                 error = 0;
 400                 snprintf(buf, sizeof(buf), "plex%d", i);
 401                 p2 = gctl_get_paraml(req, buf, sizeof(*p2));
 402                 if (p2 == NULL) {
 403                         gctl_error(req, "no plex definition given");
 404                         return (-1);
 405                 }
 406                 if (gv_find_plex(sc, p2->name) != NULL) {
 407                         /* Ignore error. */
 408                         if (*flags & GV_FLAG_F)
 409                                 continue;
 410                         gctl_error(req, "plex '%s' already exists", p2->name);
 411                         goto error;
 412                 }
 413
 414                 p = g_malloc(sizeof(*p), M_WAITOK | M_ZERO);
 415                 bcopy(p2, p, sizeof(*p));
 416
 417                 gv_post_event(sc, GV_EVENT_CREATE_PLEX, p, NULL, 0, 0);
 418         }
 419
 420         /* ... and, finally, subdisk definitions. */
 421         for (i = 0; i < *subdisks; i++) {
 422                 error = 0;
 423                 snprintf(buf, sizeof(buf), "sd%d", i);
 424                 s2 = gctl_get_paraml(req, buf, sizeof(*s2));
 425                 if (s2 == NULL) {
 426                         gctl_error(req, "no subdisk definition given");
 427                         return (-1);
 428                 }
 429                 if (gv_find_sd(sc, s2->name) != NULL) {
 430                         /* Ignore error. */
 431                         if (*flags & GV_FLAG_F)
 432                                 continue;
 433                         gctl_error(req, "sd '%s' already exists", s2->name);
 434                         goto error;
 435                 }
 436
 437                 s = g_malloc(sizeof(*s), M_WAITOK | M_ZERO);
 438                 bcopy(s2, s, sizeof(*s));
 439
 440                 gv_post_event(sc, GV_EVENT_CREATE_SD, s, NULL, 0, 0);
 441         }
 442
 443 error:
 444         gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc, NULL, 0, 0);
 445         gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
 446
 447         return (0);
 448 }
 449
 450 static void
 451 gv_config(struct gctl_req *req, struct g_class *mp, char const *verb)
 452 {
 453         struct g_geom *gp;
 454         struct gv_softc *sc;
 455         struct sbuf *sb;
 456         char *comment;
 457
 458         g_topology_assert();
 459
 460         gp = LIST_FIRST(&mp->geom);
 461         sc = gp->softc;
 462
 463         if (!strcmp(verb, "attach")) {
 464                 gv_attach(sc, req);
 465
 466         } else if (!strcmp(verb, "concat")) {
 467                 gv_concat(gp, req);
 468
 469         } else if (!strcmp(verb, "detach")) {
 470                 gv_detach(sc, req);
 471
 472         } else if (!strcmp(verb, "list")) {
 473                 gv_list(gp, req);
 474
 475         /* Save our configuration back to disk. */
 476         } else if (!strcmp(verb, "saveconfig")) {
 477                 gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
 478
 479         /* Return configuration in string form. */
 480         } else if (!strcmp(verb, "getconfig")) {
 481                 comment = gctl_get_param(req, "comment", NULL);
 482                 if (comment == NULL) {
 483                         gctl_error(req, "no comment parameter given");
 484                         return;
 485                 }
 486                 sb = sbuf_new(NULL, NULL, GV_CFG_LEN, SBUF_FIXEDLEN);
 487                 gv_format_config(sc, sb, 0, comment);
 488                 sbuf_finish(sb);
 489                 gctl_set_param(req, "config", sbuf_data(sb), sbuf_len(sb) + 1);
 490                 sbuf_delete(sb);
 491
 492         } else if (!strcmp(verb, "create")) {
 493                 gv_create(gp, req);
 494
 495         } else if (!strcmp(verb, "mirror")) {
 496                 gv_mirror(gp, req);
 497
 498         } else if (!strcmp(verb, "move")) {
 499                 gv_move(gp, req);
 500
 501         } else if (!strcmp(verb, "raid5")) {
 502                 gv_raid5(gp, req);
 503
 504         } else if (!strcmp(verb, "rebuildparity") ||
 505             !strcmp(verb, "checkparity")) {
 506                 gv_parityop(sc, req);
 507
 508         } else if (!strcmp(verb, "remove")) {
 509                 gv_remove(gp, req);
 510
 511         } else if (!strcmp(verb, "rename")) {
 512                 gv_rename(gp, req);
 513
 514         } else if (!strcmp(verb, "resetconfig")) {
 515                 gv_post_event(sc, GV_EVENT_RESET_CONFIG, sc, NULL, 0, 0);
 516
 517         } else if (!strcmp(verb, "start")) {
 518                 gv_start_obj(gp, req);
 519
 520         } else if (!strcmp(verb, "stripe")) {
 521                 gv_stripe(gp, req);
 522
 523         } else if (!strcmp(verb, "setstate")) {
 524                 gv_setstate(gp, req);
 525         } else
 526                 gctl_error(req, "Unknown verb parameter");
 527 }
 528
 529 static void
 530 gv_parityop(struct gv_softc *sc, struct gctl_req *req)
 531 {
 532         struct gv_plex *p;
 533         int *flags, *rebuild, type;
 534         char *plex;
 535
 536         plex = gctl_get_param(req, "plex", NULL);
 537         if (plex == NULL) {
 538                 gctl_error(req, "no plex given");
 539                 return;
 540         }
 541
 542         flags = gctl_get_paraml(req, "flags", sizeof(*flags));
 543         if (flags == NULL) {
 544                 gctl_error(req, "no flags given");
 545                 return;
 546         }
 547
 548         rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild));
 549         if (rebuild == NULL) {
 550                 gctl_error(req, "no operation given");
 551                 return;
 552         }
 553
 554         type = gv_object_type(sc, plex);
 555         if (type != GV_TYPE_PLEX) {
 556                 gctl_error(req, "'%s' is not a plex", plex);
 557                 return;
 558         }
 559         p = gv_find_plex(sc, plex);
 560
 561         if (p->state != GV_PLEX_UP) {
 562                 gctl_error(req, "plex %s is not completely accessible",
 563                     p->name);
 564                 return;
 565         }
 566
 567         if (p->org != GV_PLEX_RAID5) {
 568                 gctl_error(req, "plex %s is not a RAID5 plex", p->name);
 569                 return;
 570         }
 571
 572         /* Put it in the event queue. */
 573         /* XXX: The state of the plex might have changed when this event is
 574          * picked up ... We should perhaps check this afterwards. */
 575         if (*rebuild)
 576                 gv_post_event(sc, GV_EVENT_PARITY_REBUILD, p, NULL, 0, 0);
 577         else
 578                 gv_post_event(sc, GV_EVENT_PARITY_CHECK, p, NULL, 0, 0);
 579 }
 580
 581
 582 static struct g_geom *
 583 gv_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 584 {
 585         struct g_geom *gp;
 586         struct g_consumer *cp;
 587         struct gv_softc *sc;
 588         struct gv_hdr vhdr;
 589         int error;
 590
 591         g_topology_assert();
 592         g_trace(G_T_TOPOLOGY, "gv_taste(%s, %s)", mp->name, pp->name);
 593
 594         gp = LIST_FIRST(&mp->geom);
 595         if (gp == NULL) {
 596                 G_VINUM_DEBUG(0, "error: tasting, but not initialized?");
 597                 return (NULL);
 598         }
 599         sc = gp->softc;
 600
 601         cp = g_new_consumer(gp);
 602         if (g_attach(cp, pp) != 0) {
 603                 g_destroy_consumer(cp);
 604                 return (NULL);
 605         }
 606         if (g_access(cp, 1, 0, 0) != 0) {
 607                 g_detach(cp);
 608                 g_destroy_consumer(cp);
 609                 return (NULL);
 610         }
 611         g_topology_unlock();
 612
 613         error = gv_read_header(cp, &vhdr);
 614
 615         g_topology_lock();
 616         g_access(cp, -1, 0, 0);
 617         g_detach(cp);
 618         g_destroy_consumer(cp);
 619
 620         /* Check if what we've been given is a valid vinum drive. */
 621         if (!error)
 622                 gv_post_event(sc, GV_EVENT_DRIVE_TASTED, pp, NULL, 0, 0);
 623
 624         return (NULL);
 625 }
 626
 627 void
 628 gv_worker(void *arg)
 629 {
 630         struct g_provider *pp;
 631         struct gv_softc *sc;
 632         struct gv_event *ev;
 633         struct gv_volume *v;
 634         struct gv_plex *p;
 635         struct gv_sd *s;
 636         struct gv_drive *d;
 637         struct bio *bp;
 638         int newstate, flags, err, rename;
 639         char *newname;
 640         off_t offset;
 641
 642         sc = arg;
 643         KASSERT(sc != NULL, ("NULL sc"));
 644         for (;;) {
 645                 /* Look at the events first... */
 646                 ev = gv_get_event(sc);
 647                 if (ev != NULL) {
 648                         gv_remove_event(sc, ev);
 649
 650                         switch (ev->type) {
 651                         case GV_EVENT_DRIVE_TASTED:
 652                                 G_VINUM_DEBUG(2, "event 'drive tasted'");
 653                                 pp = ev->arg1;
 654                                 gv_drive_tasted(sc, pp);
 655                                 break;
 656
 657                         case GV_EVENT_DRIVE_LOST:
 658                                 G_VINUM_DEBUG(2, "event 'drive lost'");
 659                                 d = ev->arg1;
 660                                 gv_drive_lost(sc, d);
 661                                 break;
 662
 663                         case GV_EVENT_CREATE_DRIVE:
 664                                 G_VINUM_DEBUG(2, "event 'create drive'");
 665                                 d = ev->arg1;
 666                                 gv_create_drive(sc, d);
 667                                 break;
 668
 669                         case GV_EVENT_CREATE_VOLUME:
 670                                 G_VINUM_DEBUG(2, "event 'create volume'");
 671                                 v = ev->arg1;
 672                                 gv_create_volume(sc, v);
 673                                 break;
 674
 675                         case GV_EVENT_CREATE_PLEX:
 676                                 G_VINUM_DEBUG(2, "event 'create plex'");
 677                                 p = ev->arg1;
 678                                 gv_create_plex(sc, p);
 679                                 break;
 680
 681                         case GV_EVENT_CREATE_SD:
 682                                 G_VINUM_DEBUG(2, "event 'create sd'");
 683                                 s = ev->arg1;
 684                                 gv_create_sd(sc, s);
 685                                 break;
 686
 687                         case GV_EVENT_RM_DRIVE:
 688                                 G_VINUM_DEBUG(2, "event 'remove drive'");
 689                                 d = ev->arg1;
 690                                 flags = ev->arg3;
 691                                 gv_rm_drive(sc, d, flags);
 692                                 /*gv_setup_objects(sc);*/
 693                                 break;
 694
 695                         case GV_EVENT_RM_VOLUME:
 696                                 G_VINUM_DEBUG(2, "event 'remove volume'");
 697                                 v = ev->arg1;
 698                                 gv_rm_vol(sc, v);
 699                                 /*gv_setup_objects(sc);*/
 700                                 break;
 701
 702                         case GV_EVENT_RM_PLEX:
 703                                 G_VINUM_DEBUG(2, "event 'remove plex'");
 704                                 p = ev->arg1;
 705                                 gv_rm_plex(sc, p);
 706                                 /*gv_setup_objects(sc);*/
 707                                 break;
 708
 709                         case GV_EVENT_RM_SD:
 710                                 G_VINUM_DEBUG(2, "event 'remove sd'");
 711                                 s = ev->arg1;
 712                                 gv_rm_sd(sc, s);
 713                                 /*gv_setup_objects(sc);*/
 714                                 break;
 715
 716                         case GV_EVENT_SAVE_CONFIG:
 717                                 G_VINUM_DEBUG(2, "event 'save config'");
 718                                 gv_save_config(sc);
 719                                 break;
 720
 721                         case GV_EVENT_SET_SD_STATE:
 722                                 G_VINUM_DEBUG(2, "event 'setstate sd'");
 723                                 s = ev->arg1;
 724                                 newstate = ev->arg3;
 725                                 flags = ev->arg4;
 726                                 err = gv_set_sd_state(s, newstate, flags);
 727                                 if (err)
 728                                         G_VINUM_DEBUG(0, "error setting subdisk"
 729                                             " state: error code %d", err);
 730                                 break;
 731
 732                         case GV_EVENT_SET_DRIVE_STATE:
 733                                 G_VINUM_DEBUG(2, "event 'setstate drive'");
 734                                 d = ev->arg1;
 735                                 newstate = ev->arg3;
 736                                 flags = ev->arg4;
 737                                 err = gv_set_drive_state(d, newstate, flags);
 738                                 if (err)
 739                                         G_VINUM_DEBUG(0, "error setting drive "
 740                                             "state: error code %d", err);
 741                                 break;
 742
 743                         case GV_EVENT_SET_VOL_STATE:
 744                                 G_VINUM_DEBUG(2, "event 'setstate volume'");
 745                                 v = ev->arg1;
 746                                 newstate = ev->arg3;
 747                                 flags = ev->arg4;
 748                                 err = gv_set_vol_state(v, newstate, flags);
 749                                 if (err)
 750                                         G_VINUM_DEBUG(0, "error setting volume "
 751                                             "state: error code %d", err);
 752                                 break;
 753
 754                         case GV_EVENT_SET_PLEX_STATE:
 755                                 G_VINUM_DEBUG(2, "event 'setstate plex'");
 756                                 p = ev->arg1;
 757                                 newstate = ev->arg3;
 758                                 flags = ev->arg4;
 759                                 err = gv_set_plex_state(p, newstate, flags);
 760                                 if (err)
 761                                         G_VINUM_DEBUG(0, "error setting plex "
 762                                             "state: error code %d", err);
 763                                 break;
 764
 765                         case GV_EVENT_SETUP_OBJECTS:
 766                                 G_VINUM_DEBUG(2, "event 'setup objects'");
 767                                 gv_setup_objects(sc);
 768                                 break;
 769
 770                         case GV_EVENT_RESET_CONFIG:
 771                                 G_VINUM_DEBUG(2, "event 'resetconfig'");
 772                                 err = gv_resetconfig(sc);
 773                                 if (err)
 774                                         G_VINUM_DEBUG(0, "error resetting "
 775                                             "config: error code %d", err);
 776                                 break;
 777
 778                         case GV_EVENT_PARITY_REBUILD:
 779                                 /*
 780                                  * Start the rebuild. The gv_plex_done will
 781                                  * handle issuing of the remaining rebuild bio's
 782                                  * until it's finished.
 783                                  */
 784                                 G_VINUM_DEBUG(2, "event 'rebuild'");
 785                                 p = ev->arg1;
 786                                 if (p->state != GV_PLEX_UP) {
 787                                         G_VINUM_DEBUG(0, "plex %s is not "
 788                                             "completely accessible", p->name);
 789                                         break;
 790                                 }
 791                                 if (p->flags & GV_PLEX_SYNCING ||
 792                                     p->flags & GV_PLEX_REBUILDING ||
 793                                     p->flags & GV_PLEX_GROWING) {
 794                                         G_VINUM_DEBUG(0, "plex %s is busy with "
 795                                             "syncing or parity build", p->name);
 796                                         break;
 797                                 }
 798                                 p->synced = 0;
 799                                 p->flags |= GV_PLEX_REBUILDING;
 800                                 g_topology_assert_not();
 801                                 g_topology_lock();
 802                                 err = gv_access(p->vol_sc->provider, 1, 1, 0);
 803                                 if (err) {
 804                                         G_VINUM_DEBUG(0, "unable to access "
 805                                             "provider");
 806                                         break;
 807                                 }
 808                                 g_topology_unlock();
 809                                 gv_parity_request(p, GV_BIO_CHECK |
 810                                     GV_BIO_PARITY, 0);
 811                                 break;
 812
 813                         case GV_EVENT_PARITY_CHECK:
 814                                 /* Start parity check. */
 815                                 G_VINUM_DEBUG(2, "event 'check'");
 816                                 p = ev->arg1;
 817                                 if (p->state != GV_PLEX_UP) {
 818                                         G_VINUM_DEBUG(0, "plex %s is not "
 819                                             "completely accessible", p->name);
 820                                         break;
 821                                 }
 822                                 if (p->flags & GV_PLEX_SYNCING ||
 823                                     p->flags & GV_PLEX_REBUILDING ||
 824                                     p->flags & GV_PLEX_GROWING) {
 825                                         G_VINUM_DEBUG(0, "plex %s is busy with "
 826                                             "syncing or parity build", p->name);
 827                                         break;
 828                                 }
 829                                 p->synced = 0;
 830                                 g_topology_assert_not();
 831                                 g_topology_lock();
 832                                 err = gv_access(p->vol_sc->provider, 1, 1, 0);
 833                                 if (err) {
 834                                         G_VINUM_DEBUG(0, "unable to access "
 835                                             "provider");
 836                                         break;
 837                                 }
 838                                 g_topology_unlock();
 839                                 gv_parity_request(p, GV_BIO_CHECK, 0);
 840                                 break;
 841
 842                         case GV_EVENT_START_PLEX:
 843                                 G_VINUM_DEBUG(2, "event 'start' plex");
 844                                 p = ev->arg1;
 845                                 gv_start_plex(p);
 846                                 break;
 847
 848                         case GV_EVENT_START_VOLUME:
 849                                 G_VINUM_DEBUG(2, "event 'start' volume");
 850                                 v = ev->arg1;
 851                                 gv_start_vol(v);
 852                                 break;
 853
 854                         case GV_EVENT_ATTACH_PLEX:
 855                                 G_VINUM_DEBUG(2, "event 'attach' plex");
 856                                 p = ev->arg1;
 857                                 v = ev->arg2;
 858                                 rename = ev->arg4;
 859                                 err = gv_attach_plex(p, v, rename);
 860                                 if (err)
 861                                         G_VINUM_DEBUG(0, "error attaching %s to"
 862                                             " %s: error code %d", p->name,
 863                                             v->name, err);
 864                                 break;
 865
 866                         case GV_EVENT_ATTACH_SD:
 867                                 G_VINUM_DEBUG(2, "event 'attach' sd");
 868                                 s = ev->arg1;
 869                                 p = ev->arg2;
 870                                 offset = ev->arg3;
 871                                 rename = ev->arg4;
 872                                 err = gv_attach_sd(s, p, offset, rename);
 873                                 if (err)
 874                                         G_VINUM_DEBUG(0, "error attaching %s to"
 875                                             " %s: error code %d", s->name,
 876                                             p->name, err);
 877                                 break;
 878
 879                         case GV_EVENT_DETACH_PLEX:
 880                                 G_VINUM_DEBUG(2, "event 'detach' plex");
 881                                 p = ev->arg1;
 882                                 flags = ev->arg3;
 883                                 err = gv_detach_plex(p, flags);
 884                                 if (err)
 885                                         G_VINUM_DEBUG(0, "error detaching %s: "
 886                                             "error code %d", p->name, err);
 887                                 break;
 888
 889                         case GV_EVENT_DETACH_SD:
 890                                 G_VINUM_DEBUG(2, "event 'detach' sd");
 891                                 s = ev->arg1;
 892                                 flags = ev->arg3;
 893                                 err = gv_detach_sd(s, flags);
 894                                 if (err)
 895                                         G_VINUM_DEBUG(0, "error detaching %s: "
 896                                             "error code %d", s->name, err);
 897                                 break;
 898
 899                         case GV_EVENT_RENAME_VOL:
 900                                 G_VINUM_DEBUG(2, "event 'rename' volume");
 901                                 v = ev->arg1;
 902                                 newname = ev->arg2;
 903                                 flags = ev->arg3;
 904                                 err = gv_rename_vol(sc, v, newname, flags);
 905                                 if (err)
 906                                         G_VINUM_DEBUG(0, "error renaming %s to "
 907                                             "%s: error code %d", v->name,
 908                                             newname, err);
 909                                 g_free(newname);
 910                                 /* Destroy and recreate the provider if we can. */
 911                                 if (gv_provider_is_open(v->provider)) {
 912                                         G_VINUM_DEBUG(0, "unable to rename "
 913                                             "provider to %s: provider in use",
 914                                             v->name);
 915                                         break;
 916                                 }
 917                                 g_topology_lock();
 918                                 g_wither_provider(v->provider, ENOENT);
 919                                 g_topology_unlock();
 920                                 v->provider = NULL;
 921                                 gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc,
 922                                     NULL, 0, 0);
 923                                 break;
 924
 925                         case GV_EVENT_RENAME_PLEX:
 926                                 G_VINUM_DEBUG(2, "event 'rename' plex");
 927                                 p = ev->arg1;
 928                                 newname = ev->arg2;
 929                                 flags = ev->arg3;
 930                                 err = gv_rename_plex(sc, p, newname, flags);
 931                                 if (err)
 932                                         G_VINUM_DEBUG(0, "error renaming %s to "
 933                                             "%s: error code %d", p->name,
 934                                             newname, err);
 935                                 g_free(newname);
 936                                 break;
 937
 938                         case GV_EVENT_RENAME_SD:
 939                                 G_VINUM_DEBUG(2, "event 'rename' sd");
 940                                 s = ev->arg1;
 941                                 newname = ev->arg2;
 942                                 flags = ev->arg3;
 943                                 err = gv_rename_sd(sc, s, newname, flags);
 944                                 if (err)
 945                                         G_VINUM_DEBUG(0, "error renaming %s to "
 946                                             "%s: error code %d", s->name,
 947                                             newname, err);
 948                                 g_free(newname);
 949                                 break;
 950
 951                         case GV_EVENT_RENAME_DRIVE:
 952                                 G_VINUM_DEBUG(2, "event 'rename' drive");
 953                                 d = ev->arg1;
 954                                 newname = ev->arg2;
 955                                 flags = ev->arg3;
 956                                 err = gv_rename_drive(sc, d, newname, flags);
 957                                 if (err)
 958                                         G_VINUM_DEBUG(0, "error renaming %s to "
 959                                             "%s: error code %d", d->name,
 960                                             newname, err);
 961                                 g_free(newname);
 962                                 break;
 963
 964                         case GV_EVENT_MOVE_SD:
 965                                 G_VINUM_DEBUG(2, "event 'move' sd");
 966                                 s = ev->arg1;
 967                                 d = ev->arg2;
 968                                 flags = ev->arg3;
 969                                 err = gv_move_sd(sc, s, d, flags);
 970                                 if (err)
 971                                         G_VINUM_DEBUG(0, "error moving %s to "
 972                                             "%s: error code %d", s->name,
 973                                             d->name, err);
 974                                 break;
 975
 976                         case GV_EVENT_THREAD_EXIT:
 977                                 G_VINUM_DEBUG(2, "event 'thread exit'");
 978                                 g_free(ev);
 979                                 mtx_lock(&sc->equeue_mtx);
 980                                 mtx_lock(&sc->bqueue_mtx);
 981                                 gv_cleanup(sc);
 982                                 mtx_destroy(&sc->bqueue_mtx);
 983                                 mtx_destroy(&sc->equeue_mtx);
 984                                 g_free(sc->bqueue_down);
 985                                 g_free(sc->bqueue_up);
 986                                 g_free(sc);
 987                                 kproc_exit(0);
 988                                 /* NOTREACHED */
 989
 990                         default:
 991                                 G_VINUM_DEBUG(1, "unknown event %d", ev->type);
 992                         }
 993
 994                         g_free(ev);
 995                         continue;
 996                 }
 997
 998                 /* ... then do I/O processing. */
 999                 mtx_lock(&sc->bqueue_mtx);
1000                 /* First do new requests. */
1001                 bp = bioq_takefirst(sc->bqueue_down);
1002                 if (bp != NULL) {
1003                         mtx_unlock(&sc->bqueue_mtx);
1004                         /* A bio that interfered with another bio. */
1005                         if (bp->bio_pflags & GV_BIO_ONHOLD) {
1006                                 s = bp->bio_caller1;
1007                                 p = s->plex_sc;
1008                                 /* Is it still locked out? */
1009                                 if (gv_stripe_active(p, bp)) {
1010                                         /* Park the bio on the waiting queue. */
1011                                         bioq_disksort(p->wqueue, bp);
1012                                 } else {
1013                                         bp->bio_pflags &= ~GV_BIO_ONHOLD;
1014                                         g_io_request(bp, s->drive_sc->consumer);
1015                                 }
1016                         /* A special request requireing special handling. */
1017                         } else if (bp->bio_pflags & GV_BIO_INTERNAL) {
1018                                 p = bp->bio_caller1;
1019                                 gv_plex_start(p, bp);
1020                         } else {
1021                                 gv_volume_start(sc, bp);
1022                         }
1023                         mtx_lock(&sc->bqueue_mtx);
1024                 }
1025                 /* Then do completed requests. */
1026                 bp = bioq_takefirst(sc->bqueue_up);
1027                 if (bp == NULL) {
1028                         msleep(sc, &sc->bqueue_mtx, PRIBIO, "-", hz/10);
1029                         mtx_unlock(&sc->bqueue_mtx);
1030                         continue;
1031                 }
1032                 mtx_unlock(&sc->bqueue_mtx);
1033                 gv_bio_done(sc, bp);
1034         }
1035 }
1036
1037 #define VINUM_CLASS_NAME "VINUM"
1038
1039 static struct g_class g_vinum_class     = {
1040         .name = VINUM_CLASS_NAME,
1041         .version = G_VERSION,
1042         .init = gv_init,
1043         .taste = gv_taste,
1044         .ctlreq = gv_config,
1045         .destroy_geom = gv_unload,
1046 };
1047
1048 DECLARE_GEOM_CLASS(g_vinum_class, g_vinum);