sys/geom/mirror/g_mirror.c

   1 /*-
   2  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/module.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/bio.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/malloc.h>
  41 #include <sys/eventhandler.h>
  42 #include <vm/uma.h>
  43 #include <geom/geom.h>
  44 #include <sys/proc.h>
  45 #include <sys/kthread.h>
  46 #include <sys/sched.h>
  47 #include <geom/mirror/g_mirror.h>
  48
  49 FEATURE(geom_mirror, "GEOM mirroring support");
  50
  51 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
  52
  53 SYSCTL_DECL(_kern_geom);
  54 static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
  55     "GEOM_MIRROR stuff");
  56 u_int g_mirror_debug = 0;
  57 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RWTUN, &g_mirror_debug, 0,
  58     "Debug level");
  59 static u_int g_mirror_timeout = 4;
  60 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RWTUN, &g_mirror_timeout,
  61     0, "Time to wait on all mirror components");
  62 static u_int g_mirror_idletime = 5;
  63 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RWTUN,
  64     &g_mirror_idletime, 0, "Mark components as clean when idling");
  65 static u_int g_mirror_disconnect_on_failure = 1;
  66 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RWTUN,
  67     &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  68 static u_int g_mirror_syncreqs = 2;
  69 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  70     &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
  71
  72 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  73         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));       \
  74         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  75         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));       \
  76 } while (0)
  77
  78 static eventhandler_tag g_mirror_post_sync = NULL;
  79 static int g_mirror_shutdown = 0;
  80
  81 static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
  82     struct g_geom *gp);
  83 static g_taste_t g_mirror_taste;
  84 static g_resize_t g_mirror_resize;
  85 static void g_mirror_init(struct g_class *mp);
  86 static void g_mirror_fini(struct g_class *mp);
  87
  88 struct g_class g_mirror_class = {
  89         .name = G_MIRROR_CLASS_NAME,
  90         .version = G_VERSION,
  91         .ctlreq = g_mirror_config,
  92         .taste = g_mirror_taste,
  93         .destroy_geom = g_mirror_destroy_geom,
  94         .init = g_mirror_init,
  95         .fini = g_mirror_fini,
  96         .resize = g_mirror_resize
  97 };
  98
  99
 100 static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
 101 static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
 102 static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
 103 static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
 104     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 105 static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
 106 static void g_mirror_register_request(struct bio *bp);
 107 static void g_mirror_sync_release(struct g_mirror_softc *sc);
 108
 109
 110 static const char *
 111 g_mirror_disk_state2str(int state)
 112 {
 113
 114         switch (state) {
 115         case G_MIRROR_DISK_STATE_NONE:
 116                 return ("NONE");
 117         case G_MIRROR_DISK_STATE_NEW:
 118                 return ("NEW");
 119         case G_MIRROR_DISK_STATE_ACTIVE:
 120                 return ("ACTIVE");
 121         case G_MIRROR_DISK_STATE_STALE:
 122                 return ("STALE");
 123         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 124                 return ("SYNCHRONIZING");
 125         case G_MIRROR_DISK_STATE_DISCONNECTED:
 126                 return ("DISCONNECTED");
 127         case G_MIRROR_DISK_STATE_DESTROY:
 128                 return ("DESTROY");
 129         default:
 130                 return ("INVALID");
 131         }
 132 }
 133
 134 static const char *
 135 g_mirror_device_state2str(int state)
 136 {
 137
 138         switch (state) {
 139         case G_MIRROR_DEVICE_STATE_STARTING:
 140                 return ("STARTING");
 141         case G_MIRROR_DEVICE_STATE_RUNNING:
 142                 return ("RUNNING");
 143         default:
 144                 return ("INVALID");
 145         }
 146 }
 147
 148 static const char *
 149 g_mirror_get_diskname(struct g_mirror_disk *disk)
 150 {
 151
 152         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 153                 return ("[unknown]");
 154         return (disk->d_name);
 155 }
 156
 157 /*
 158  * --- Events handling functions ---
 159  * Events in geom_mirror are used to maintain disks and device status
 160  * from one thread to simplify locking.
 161  */
 162 static void
 163 g_mirror_event_free(struct g_mirror_event *ep)
 164 {
 165
 166         free(ep, M_MIRROR);
 167 }
 168
 169 int
 170 g_mirror_event_send(void *arg, int state, int flags)
 171 {
 172         struct g_mirror_softc *sc;
 173         struct g_mirror_disk *disk;
 174         struct g_mirror_event *ep;
 175         int error;
 176
 177         ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
 178         G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 179         if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
 180                 disk = NULL;
 181                 sc = arg;
 182         } else {
 183                 disk = arg;
 184                 sc = disk->d_softc;
 185         }
 186         ep->e_disk = disk;
 187         ep->e_state = state;
 188         ep->e_flags = flags;
 189         ep->e_error = 0;
 190         mtx_lock(&sc->sc_events_mtx);
 191         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 192         mtx_unlock(&sc->sc_events_mtx);
 193         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 194         mtx_lock(&sc->sc_queue_mtx);
 195         wakeup(sc);
 196         mtx_unlock(&sc->sc_queue_mtx);
 197         if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 198                 return (0);
 199         sx_assert(&sc->sc_lock, SX_XLOCKED);
 200         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 201         sx_xunlock(&sc->sc_lock);
 202         while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
 203                 mtx_lock(&sc->sc_events_mtx);
 204                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
 205                     hz * 5);
 206         }
 207         error = ep->e_error;
 208         g_mirror_event_free(ep);
 209         sx_xlock(&sc->sc_lock);
 210         return (error);
 211 }
 212
 213 static struct g_mirror_event *
 214 g_mirror_event_get(struct g_mirror_softc *sc)
 215 {
 216         struct g_mirror_event *ep;
 217
 218         mtx_lock(&sc->sc_events_mtx);
 219         ep = TAILQ_FIRST(&sc->sc_events);
 220         mtx_unlock(&sc->sc_events_mtx);
 221         return (ep);
 222 }
 223
 224 static void
 225 g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
 226 {
 227
 228         mtx_lock(&sc->sc_events_mtx);
 229         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 230         mtx_unlock(&sc->sc_events_mtx);
 231 }
 232
 233 static void
 234 g_mirror_event_cancel(struct g_mirror_disk *disk)
 235 {
 236         struct g_mirror_softc *sc;
 237         struct g_mirror_event *ep, *tmpep;
 238
 239         sc = disk->d_softc;
 240         sx_assert(&sc->sc_lock, SX_XLOCKED);
 241
 242         mtx_lock(&sc->sc_events_mtx);
 243         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 244                 if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
 245                         continue;
 246                 if (ep->e_disk != disk)
 247                         continue;
 248                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 249                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 250                         g_mirror_event_free(ep);
 251                 else {
 252                         ep->e_error = ECANCELED;
 253                         wakeup(ep);
 254                 }
 255         }
 256         mtx_unlock(&sc->sc_events_mtx);
 257 }
 258
 259 /*
 260  * Return the number of disks in given state.
 261  * If state is equal to -1, count all connected disks.
 262  */
 263 u_int
 264 g_mirror_ndisks(struct g_mirror_softc *sc, int state)
 265 {
 266         struct g_mirror_disk *disk;
 267         u_int n = 0;
 268
 269         sx_assert(&sc->sc_lock, SX_LOCKED);
 270
 271         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 272                 if (state == -1 || disk->d_state == state)
 273                         n++;
 274         }
 275         return (n);
 276 }
 277
 278 /*
 279  * Find a disk in mirror by its disk ID.
 280  */
 281 static struct g_mirror_disk *
 282 g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
 283 {
 284         struct g_mirror_disk *disk;
 285
 286         sx_assert(&sc->sc_lock, SX_XLOCKED);
 287
 288         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 289                 if (disk->d_id == id)
 290                         return (disk);
 291         }
 292         return (NULL);
 293 }
 294
 295 static u_int
 296 g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
 297 {
 298         struct bio *bp;
 299         u_int nreqs = 0;
 300
 301         mtx_lock(&sc->sc_queue_mtx);
 302         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 303                 if (bp->bio_from == cp)
 304                         nreqs++;
 305         }
 306         mtx_unlock(&sc->sc_queue_mtx);
 307         return (nreqs);
 308 }
 309
 310 static int
 311 g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
 312 {
 313
 314         if (cp->index > 0) {
 315                 G_MIRROR_DEBUG(2,
 316                     "I/O requests for %s exist, can't destroy it now.",
 317                     cp->provider->name);
 318                 return (1);
 319         }
 320         if (g_mirror_nrequests(sc, cp) > 0) {
 321                 G_MIRROR_DEBUG(2,
 322                     "I/O requests for %s in queue, can't destroy it now.",
 323                     cp->provider->name);
 324                 return (1);
 325         }
 326         return (0);
 327 }
 328
 329 static void
 330 g_mirror_destroy_consumer(void *arg, int flags __unused)
 331 {
 332         struct g_consumer *cp;
 333
 334         g_topology_assert();
 335
 336         cp = arg;
 337         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 338         g_detach(cp);
 339         g_destroy_consumer(cp);
 340 }
 341
 342 static void
 343 g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 344 {
 345         struct g_provider *pp;
 346         int retaste_wait;
 347
 348         g_topology_assert();
 349
 350         cp->private = NULL;
 351         if (g_mirror_is_busy(sc, cp))
 352                 return;
 353         pp = cp->provider;
 354         retaste_wait = 0;
 355         if (cp->acw == 1) {
 356                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 357                         retaste_wait = 1;
 358         }
 359         G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 360             -cp->acw, -cp->ace, 0);
 361         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 362                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 363         if (retaste_wait) {
 364                 /*
 365                  * After retaste event was send (inside g_access()), we can send
 366                  * event to detach and destroy consumer.
 367                  * A class, which has consumer to the given provider connected
 368                  * will not receive retaste event for the provider.
 369                  * This is the way how I ignore retaste events when I close
 370                  * consumers opened for write: I detach and destroy consumer
 371                  * after retaste event is sent.
 372                  */
 373                 g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
 374                 return;
 375         }
 376         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
 377         g_detach(cp);
 378         g_destroy_consumer(cp);
 379 }
 380
 381 static int
 382 g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
 383 {
 384         struct g_consumer *cp;
 385         int error;
 386
 387         g_topology_assert_not();
 388         KASSERT(disk->d_consumer == NULL,
 389             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 390
 391         g_topology_lock();
 392         cp = g_new_consumer(disk->d_softc->sc_geom);
 393         cp->flags |= G_CF_DIRECT_RECEIVE;
 394         error = g_attach(cp, pp);
 395         if (error != 0) {
 396                 g_destroy_consumer(cp);
 397                 g_topology_unlock();
 398                 return (error);
 399         }
 400         error = g_access(cp, 1, 1, 1);
 401         if (error != 0) {
 402                 g_detach(cp);
 403                 g_destroy_consumer(cp);
 404                 g_topology_unlock();
 405                 G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
 406                     pp->name, error);
 407                 return (error);
 408         }
 409         g_topology_unlock();
 410         disk->d_consumer = cp;
 411         disk->d_consumer->private = disk;
 412         disk->d_consumer->index = 0;
 413
 414         G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
 415         return (0);
 416 }
 417
 418 static void
 419 g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 420 {
 421
 422         g_topology_assert();
 423
 424         if (cp == NULL)
 425                 return;
 426         if (cp->provider != NULL)
 427                 g_mirror_kill_consumer(sc, cp);
 428         else
 429                 g_destroy_consumer(cp);
 430 }
 431
 432 /*
 433  * Initialize disk. This means allocate memory, create consumer, attach it
 434  * to the provider and open access (r1w1e1) to it.
 435  */
 436 static struct g_mirror_disk *
 437 g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 438     struct g_mirror_metadata *md, int *errorp)
 439 {
 440         struct g_mirror_disk *disk;
 441         int i, error;
 442
 443         disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
 444         if (disk == NULL) {
 445                 error = ENOMEM;
 446                 goto fail;
 447         }
 448         disk->d_softc = sc;
 449         error = g_mirror_connect_disk(disk, pp);
 450         if (error != 0)
 451                 goto fail;
 452         disk->d_id = md->md_did;
 453         disk->d_state = G_MIRROR_DISK_STATE_NONE;
 454         disk->d_priority = md->md_priority;
 455         disk->d_flags = md->md_dflags;
 456         error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
 457         if (error == 0 && i != 0)
 458                 disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 459         if (md->md_provider[0] != '\0')
 460                 disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
 461         disk->d_sync.ds_consumer = NULL;
 462         disk->d_sync.ds_offset = md->md_sync_offset;
 463         disk->d_sync.ds_offset_done = md->md_sync_offset;
 464         disk->d_genid = md->md_genid;
 465         disk->d_sync.ds_syncid = md->md_syncid;
 466         if (errorp != NULL)
 467                 *errorp = 0;
 468         return (disk);
 469 fail:
 470         if (errorp != NULL)
 471                 *errorp = error;
 472         if (disk != NULL)
 473                 free(disk, M_MIRROR);
 474         return (NULL);
 475 }
 476
 477 static void
 478 g_mirror_destroy_disk(struct g_mirror_disk *disk)
 479 {
 480         struct g_mirror_softc *sc;
 481
 482         g_topology_assert_not();
 483         sc = disk->d_softc;
 484         sx_assert(&sc->sc_lock, SX_XLOCKED);
 485
 486         LIST_REMOVE(disk, d_next);
 487         g_mirror_event_cancel(disk);
 488         if (sc->sc_hint == disk)
 489                 sc->sc_hint = NULL;
 490         switch (disk->d_state) {
 491         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 492                 g_mirror_sync_stop(disk, 1);
 493                 /* FALLTHROUGH */
 494         case G_MIRROR_DISK_STATE_NEW:
 495         case G_MIRROR_DISK_STATE_STALE:
 496         case G_MIRROR_DISK_STATE_ACTIVE:
 497                 g_topology_lock();
 498                 g_mirror_disconnect_consumer(sc, disk->d_consumer);
 499                 g_topology_unlock();
 500                 free(disk, M_MIRROR);
 501                 break;
 502         default:
 503                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 504                     g_mirror_get_diskname(disk),
 505                     g_mirror_disk_state2str(disk->d_state)));
 506         }
 507 }
 508
 509 static void
 510 g_mirror_destroy_device(struct g_mirror_softc *sc)
 511 {
 512         struct g_mirror_disk *disk;
 513         struct g_mirror_event *ep;
 514         struct g_geom *gp;
 515         struct g_consumer *cp, *tmpcp;
 516
 517         g_topology_assert_not();
 518         sx_assert(&sc->sc_lock, SX_XLOCKED);
 519
 520         gp = sc->sc_geom;
 521         if (sc->sc_provider != NULL)
 522                 g_mirror_destroy_provider(sc);
 523         for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
 524             disk = LIST_FIRST(&sc->sc_disks)) {
 525                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 526                 g_mirror_update_metadata(disk);
 527                 g_mirror_destroy_disk(disk);
 528         }
 529         while ((ep = g_mirror_event_get(sc)) != NULL) {
 530                 g_mirror_event_remove(sc, ep);
 531                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 532                         g_mirror_event_free(ep);
 533                 else {
 534                         ep->e_error = ECANCELED;
 535                         ep->e_flags |= G_MIRROR_EVENT_DONE;
 536                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 537                         mtx_lock(&sc->sc_events_mtx);
 538                         wakeup(ep);
 539                         mtx_unlock(&sc->sc_events_mtx);
 540                 }
 541         }
 542         callout_drain(&sc->sc_callout);
 543
 544         g_topology_lock();
 545         LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
 546                 g_mirror_disconnect_consumer(sc, cp);
 547         }
 548         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 549         G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
 550         g_wither_geom(gp, ENXIO);
 551         g_topology_unlock();
 552         mtx_destroy(&sc->sc_queue_mtx);
 553         mtx_destroy(&sc->sc_events_mtx);
 554         mtx_destroy(&sc->sc_done_mtx);
 555         sx_xunlock(&sc->sc_lock);
 556         sx_destroy(&sc->sc_lock);
 557 }
 558
 559 static void
 560 g_mirror_orphan(struct g_consumer *cp)
 561 {
 562         struct g_mirror_disk *disk;
 563
 564         g_topology_assert();
 565
 566         disk = cp->private;
 567         if (disk == NULL)
 568                 return;
 569         disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 570         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 571             G_MIRROR_EVENT_DONTWAIT);
 572 }
 573
 574 /*
 575  * Function should return the next active disk on the list.
 576  * It is possible that it will be the same disk as given.
 577  * If there are no active disks on list, NULL is returned.
 578  */
 579 static __inline struct g_mirror_disk *
 580 g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
 581 {
 582         struct g_mirror_disk *dp;
 583
 584         for (dp = LIST_NEXT(disk, d_next); dp != disk;
 585             dp = LIST_NEXT(dp, d_next)) {
 586                 if (dp == NULL)
 587                         dp = LIST_FIRST(&sc->sc_disks);
 588                 if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
 589                         break;
 590         }
 591         if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 592                 return (NULL);
 593         return (dp);
 594 }
 595
 596 static struct g_mirror_disk *
 597 g_mirror_get_disk(struct g_mirror_softc *sc)
 598 {
 599         struct g_mirror_disk *disk;
 600
 601         if (sc->sc_hint == NULL) {
 602                 sc->sc_hint = LIST_FIRST(&sc->sc_disks);
 603                 if (sc->sc_hint == NULL)
 604                         return (NULL);
 605         }
 606         disk = sc->sc_hint;
 607         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
 608                 disk = g_mirror_find_next(sc, disk);
 609                 if (disk == NULL)
 610                         return (NULL);
 611         }
 612         sc->sc_hint = g_mirror_find_next(sc, disk);
 613         return (disk);
 614 }
 615
 616 static int
 617 g_mirror_write_metadata(struct g_mirror_disk *disk,
 618     struct g_mirror_metadata *md)
 619 {
 620         struct g_mirror_softc *sc;
 621         struct g_consumer *cp;
 622         off_t offset, length;
 623         u_char *sector;
 624         int error = 0;
 625
 626         g_topology_assert_not();
 627         sc = disk->d_softc;
 628         sx_assert(&sc->sc_lock, SX_LOCKED);
 629
 630         cp = disk->d_consumer;
 631         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 632         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 633         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 634             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 635             cp->acw, cp->ace));
 636         length = cp->provider->sectorsize;
 637         offset = cp->provider->mediasize - length;
 638         sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
 639         if (md != NULL &&
 640             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
 641                 /*
 642                  * Handle the case, when the size of parent provider reduced.
 643                  */
 644                 if (offset < md->md_mediasize)
 645                         error = ENOSPC;
 646                 else
 647                         mirror_metadata_encode(md, sector);
 648         }
 649         if (error == 0)
 650                 error = g_write_data(cp, offset, sector, length);
 651         free(sector, M_MIRROR);
 652         if (error != 0) {
 653                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 654                         disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 655                         G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
 656                             "(device=%s, error=%d).",
 657                             g_mirror_get_diskname(disk), sc->sc_name, error);
 658                 } else {
 659                         G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
 660                             "(device=%s, error=%d).",
 661                             g_mirror_get_diskname(disk), sc->sc_name, error);
 662                 }
 663                 if (g_mirror_disconnect_on_failure &&
 664                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 665                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 666                         g_mirror_event_send(disk,
 667                             G_MIRROR_DISK_STATE_DISCONNECTED,
 668                             G_MIRROR_EVENT_DONTWAIT);
 669                 }
 670         }
 671         return (error);
 672 }
 673
 674 static int
 675 g_mirror_clear_metadata(struct g_mirror_disk *disk)
 676 {
 677         int error;
 678
 679         g_topology_assert_not();
 680         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 681
 682         error = g_mirror_write_metadata(disk, NULL);
 683         if (error == 0) {
 684                 G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
 685                     g_mirror_get_diskname(disk));
 686         } else {
 687                 G_MIRROR_DEBUG(0,
 688                     "Cannot clear metadata on disk %s (error=%d).",
 689                     g_mirror_get_diskname(disk), error);
 690         }
 691         return (error);
 692 }
 693
 694 void
 695 g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
 696     struct g_mirror_metadata *md)
 697 {
 698
 699         strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
 700         md->md_version = G_MIRROR_VERSION;
 701         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 702         md->md_mid = sc->sc_id;
 703         md->md_all = sc->sc_ndisks;
 704         md->md_slice = sc->sc_slice;
 705         md->md_balance = sc->sc_balance;
 706         md->md_genid = sc->sc_genid;
 707         md->md_mediasize = sc->sc_mediasize;
 708         md->md_sectorsize = sc->sc_sectorsize;
 709         md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
 710         bzero(md->md_provider, sizeof(md->md_provider));
 711         if (disk == NULL) {
 712                 md->md_did = arc4random();
 713                 md->md_priority = 0;
 714                 md->md_syncid = 0;
 715                 md->md_dflags = 0;
 716                 md->md_sync_offset = 0;
 717                 md->md_provsize = 0;
 718         } else {
 719                 md->md_did = disk->d_id;
 720                 md->md_priority = disk->d_priority;
 721                 md->md_syncid = disk->d_sync.ds_syncid;
 722                 md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
 723                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
 724                         md->md_sync_offset = disk->d_sync.ds_offset_done;
 725                 else
 726                         md->md_sync_offset = 0;
 727                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
 728                         strlcpy(md->md_provider,
 729                             disk->d_consumer->provider->name,
 730                             sizeof(md->md_provider));
 731                 }
 732                 md->md_provsize = disk->d_consumer->provider->mediasize;
 733         }
 734 }
 735
 736 void
 737 g_mirror_update_metadata(struct g_mirror_disk *disk)
 738 {
 739         struct g_mirror_softc *sc;
 740         struct g_mirror_metadata md;
 741         int error;
 742
 743         g_topology_assert_not();
 744         sc = disk->d_softc;
 745         sx_assert(&sc->sc_lock, SX_LOCKED);
 746
 747         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
 748                 g_mirror_fill_metadata(sc, disk, &md);
 749         error = g_mirror_write_metadata(disk, &md);
 750         if (error == 0) {
 751                 G_MIRROR_DEBUG(2, "Metadata on %s updated.",
 752                     g_mirror_get_diskname(disk));
 753         } else {
 754                 G_MIRROR_DEBUG(0,
 755                     "Cannot update metadata on disk %s (error=%d).",
 756                     g_mirror_get_diskname(disk), error);
 757         }
 758 }
 759
 760 static void
 761 g_mirror_bump_syncid(struct g_mirror_softc *sc)
 762 {
 763         struct g_mirror_disk *disk;
 764
 765         g_topology_assert_not();
 766         sx_assert(&sc->sc_lock, SX_XLOCKED);
 767         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 768             ("%s called with no active disks (device=%s).", __func__,
 769             sc->sc_name));
 770
 771         sc->sc_syncid++;
 772         G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 773             sc->sc_syncid);
 774         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 775                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 776                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 777                         disk->d_sync.ds_syncid = sc->sc_syncid;
 778                         g_mirror_update_metadata(disk);
 779                 }
 780         }
 781 }
 782
 783 static void
 784 g_mirror_bump_genid(struct g_mirror_softc *sc)
 785 {
 786         struct g_mirror_disk *disk;
 787
 788         g_topology_assert_not();
 789         sx_assert(&sc->sc_lock, SX_XLOCKED);
 790         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 791             ("%s called with no active disks (device=%s).", __func__,
 792             sc->sc_name));
 793
 794         sc->sc_genid++;
 795         G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 796             sc->sc_genid);
 797         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 798                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 799                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 800                         disk->d_genid = sc->sc_genid;
 801                         g_mirror_update_metadata(disk);
 802                 }
 803         }
 804 }
 805
 806 static int
 807 g_mirror_idle(struct g_mirror_softc *sc, int acw)
 808 {
 809         struct g_mirror_disk *disk;
 810         int timeout;
 811
 812         g_topology_assert_not();
 813         sx_assert(&sc->sc_lock, SX_XLOCKED);
 814
 815         if (sc->sc_provider == NULL)
 816                 return (0);
 817         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 818                 return (0);
 819         if (sc->sc_idle)
 820                 return (0);
 821         if (sc->sc_writes > 0)
 822                 return (0);
 823         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 824                 timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
 825                 if (!g_mirror_shutdown && timeout > 0)
 826                         return (timeout);
 827         }
 828         sc->sc_idle = 1;
 829         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 830                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 831                         continue;
 832                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as clean.",
 833                     g_mirror_get_diskname(disk), sc->sc_name);
 834                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 835                 g_mirror_update_metadata(disk);
 836         }
 837         return (0);
 838 }
 839
 840 static void
 841 g_mirror_unidle(struct g_mirror_softc *sc)
 842 {
 843         struct g_mirror_disk *disk;
 844
 845         g_topology_assert_not();
 846         sx_assert(&sc->sc_lock, SX_XLOCKED);
 847
 848         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 849                 return;
 850         sc->sc_idle = 0;
 851         sc->sc_last_write = time_uptime;
 852         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 853                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 854                         continue;
 855                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as dirty.",
 856                     g_mirror_get_diskname(disk), sc->sc_name);
 857                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
 858                 g_mirror_update_metadata(disk);
 859         }
 860 }
 861
 862 static void
 863 g_mirror_flush_done(struct bio *bp)
 864 {
 865         struct g_mirror_softc *sc;
 866         struct bio *pbp;
 867
 868         pbp = bp->bio_parent;
 869         sc = pbp->bio_to->geom->softc;
 870         mtx_lock(&sc->sc_done_mtx);
 871         if (pbp->bio_error == 0)
 872                 pbp->bio_error = bp->bio_error;
 873         pbp->bio_completed += bp->bio_completed;
 874         pbp->bio_inbed++;
 875         if (pbp->bio_children == pbp->bio_inbed) {
 876                 mtx_unlock(&sc->sc_done_mtx);
 877                 g_io_deliver(pbp, pbp->bio_error);
 878         } else
 879                 mtx_unlock(&sc->sc_done_mtx);
 880         g_destroy_bio(bp);
 881 }
 882
 883 static void
 884 g_mirror_done(struct bio *bp)
 885 {
 886         struct g_mirror_softc *sc;
 887
 888         sc = bp->bio_from->geom->softc;
 889         bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
 890         mtx_lock(&sc->sc_queue_mtx);
 891         bioq_insert_tail(&sc->sc_queue, bp);
 892         mtx_unlock(&sc->sc_queue_mtx);
 893         wakeup(sc);
 894 }
 895
 896 static void
 897 g_mirror_regular_request(struct bio *bp)
 898 {
 899         struct g_mirror_softc *sc;
 900         struct g_mirror_disk *disk;
 901         struct bio *pbp;
 902
 903         g_topology_assert_not();
 904
 905         pbp = bp->bio_parent;
 906         sc = pbp->bio_to->geom->softc;
 907         bp->bio_from->index--;
 908         if (bp->bio_cmd == BIO_WRITE)
 909                 sc->sc_writes--;
 910         disk = bp->bio_from->private;
 911         if (disk == NULL) {
 912                 g_topology_lock();
 913                 g_mirror_kill_consumer(sc, bp->bio_from);
 914                 g_topology_unlock();
 915         }
 916
 917         pbp->bio_inbed++;
 918         KASSERT(pbp->bio_inbed <= pbp->bio_children,
 919             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
 920             pbp->bio_children));
 921         if (bp->bio_error == 0 && pbp->bio_error == 0) {
 922                 G_MIRROR_LOGREQ(3, bp, "Request delivered.");
 923                 g_destroy_bio(bp);
 924                 if (pbp->bio_children == pbp->bio_inbed) {
 925                         G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
 926                         pbp->bio_completed = pbp->bio_length;
 927                         if (pbp->bio_cmd == BIO_WRITE ||
 928                             pbp->bio_cmd == BIO_DELETE) {
 929                                 bioq_remove(&sc->sc_inflight, pbp);
 930                                 /* Release delayed sync requests if possible. */
 931                                 g_mirror_sync_release(sc);
 932                         }
 933                         g_io_deliver(pbp, pbp->bio_error);
 934                 }
 935                 return;
 936         } else if (bp->bio_error != 0) {
 937                 if (pbp->bio_error == 0)
 938                         pbp->bio_error = bp->bio_error;
 939                 if (disk != NULL) {
 940                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 941                                 disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 942                                 G_MIRROR_LOGREQ(0, bp,
 943                                     "Request failed (error=%d).",
 944                                     bp->bio_error);
 945                         } else {
 946                                 G_MIRROR_LOGREQ(1, bp,
 947                                     "Request failed (error=%d).",
 948                                     bp->bio_error);
 949                         }
 950                         if (g_mirror_disconnect_on_failure &&
 951                             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
 952                         {
 953                                 sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 954                                 g_mirror_event_send(disk,
 955                                     G_MIRROR_DISK_STATE_DISCONNECTED,
 956                                     G_MIRROR_EVENT_DONTWAIT);
 957                         }
 958                 }
 959                 switch (pbp->bio_cmd) {
 960                 case BIO_DELETE:
 961                 case BIO_WRITE:
 962                         pbp->bio_inbed--;
 963                         pbp->bio_children--;
 964                         break;
 965                 }
 966         }
 967         g_destroy_bio(bp);
 968
 969         switch (pbp->bio_cmd) {
 970         case BIO_READ:
 971                 if (pbp->bio_inbed < pbp->bio_children)
 972                         break;
 973                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
 974                         g_io_deliver(pbp, pbp->bio_error);
 975                 else {
 976                         pbp->bio_error = 0;
 977                         mtx_lock(&sc->sc_queue_mtx);
 978                         bioq_insert_tail(&sc->sc_queue, pbp);
 979                         mtx_unlock(&sc->sc_queue_mtx);
 980                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 981                         wakeup(sc);
 982                 }
 983                 break;
 984         case BIO_DELETE:
 985         case BIO_WRITE:
 986                 if (pbp->bio_children == 0) {
 987                         /*
 988                          * All requests failed.
 989                          */
 990                 } else if (pbp->bio_inbed < pbp->bio_children) {
 991                         /* Do nothing. */
 992                         break;
 993                 } else if (pbp->bio_children == pbp->bio_inbed) {
 994                         /* Some requests succeeded. */
 995                         pbp->bio_error = 0;
 996                         pbp->bio_completed = pbp->bio_length;
 997                 }
 998                 bioq_remove(&sc->sc_inflight, pbp);
 999                 /* Release delayed sync requests if possible. */
1000                 g_mirror_sync_release(sc);
1001                 g_io_deliver(pbp, pbp->bio_error);
1002                 break;
1003         default:
1004                 KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1005                 break;
1006         }
1007 }
1008
1009 static void
1010 g_mirror_sync_done(struct bio *bp)
1011 {
1012         struct g_mirror_softc *sc;
1013
1014         G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1015         sc = bp->bio_from->geom->softc;
1016         bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1017         mtx_lock(&sc->sc_queue_mtx);
1018         bioq_insert_tail(&sc->sc_queue, bp);
1019         mtx_unlock(&sc->sc_queue_mtx);
1020         wakeup(sc);
1021 }
1022
1023 static void
1024 g_mirror_candelete(struct bio *bp)
1025 {
1026         struct g_mirror_softc *sc;
1027         struct g_mirror_disk *disk;
1028         int *val;
1029
1030         sc = bp->bio_to->geom->softc;
1031         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1032                 if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
1033                         break;
1034         }
1035         val = (int *)bp->bio_data;
1036         *val = (disk != NULL);
1037         g_io_deliver(bp, 0);
1038 }
1039
1040 static void
1041 g_mirror_kernel_dump(struct bio *bp)
1042 {
1043         struct g_mirror_softc *sc;
1044         struct g_mirror_disk *disk;
1045         struct bio *cbp;
1046         struct g_kerneldump *gkd;
1047
1048         /*
1049          * We configure dumping to the first component, because this component
1050          * will be used for reading with 'prefer' balance algorithm.
1051          * If the component with the highest priority is currently disconnected
1052          * we will not be able to read the dump after the reboot if it will be
1053          * connected and synchronized later. Can we do something better?
1054          */
1055         sc = bp->bio_to->geom->softc;
1056         disk = LIST_FIRST(&sc->sc_disks);
1057
1058         gkd = (struct g_kerneldump *)bp->bio_data;
1059         if (gkd->length > bp->bio_to->mediasize)
1060                 gkd->length = bp->bio_to->mediasize;
1061         cbp = g_clone_bio(bp);
1062         if (cbp == NULL) {
1063                 g_io_deliver(bp, ENOMEM);
1064                 return;
1065         }
1066         cbp->bio_done = g_std_done;
1067         g_io_request(cbp, disk->d_consumer);
1068         G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1069             g_mirror_get_diskname(disk));
1070 }
1071
1072 static void
1073 g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
1074 {
1075         struct bio_queue_head queue;
1076         struct g_mirror_disk *disk;
1077         struct g_consumer *cp;
1078         struct bio *cbp;
1079
1080         bioq_init(&queue);
1081         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1082                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1083                         continue;
1084                 cbp = g_clone_bio(bp);
1085                 if (cbp == NULL) {
1086                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1087                                 g_destroy_bio(cbp);
1088                         if (bp->bio_error == 0)
1089                                 bp->bio_error = ENOMEM;
1090                         g_io_deliver(bp, bp->bio_error);
1091                         return;
1092                 }
1093                 bioq_insert_tail(&queue, cbp);
1094                 cbp->bio_done = g_mirror_flush_done;
1095                 cbp->bio_caller1 = disk;
1096                 cbp->bio_to = disk->d_consumer->provider;
1097         }
1098         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1099                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1100                 disk = cbp->bio_caller1;
1101                 cbp->bio_caller1 = NULL;
1102                 cp = disk->d_consumer;
1103                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1104                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1105                     cp->acr, cp->acw, cp->ace));
1106                 g_io_request(cbp, disk->d_consumer);
1107         }
1108 }
1109
1110 static void
1111 g_mirror_start(struct bio *bp)
1112 {
1113         struct g_mirror_softc *sc;
1114
1115         sc = bp->bio_to->geom->softc;
1116         /*
1117          * If sc == NULL or there are no valid disks, provider's error
1118          * should be set and g_mirror_start() should not be called at all.
1119          */
1120         KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1121             ("Provider's error should be set (error=%d)(mirror=%s).",
1122             bp->bio_to->error, bp->bio_to->name));
1123         G_MIRROR_LOGREQ(3, bp, "Request received.");
1124
1125         switch (bp->bio_cmd) {
1126         case BIO_READ:
1127         case BIO_WRITE:
1128         case BIO_DELETE:
1129                 break;
1130         case BIO_FLUSH:
1131                 g_mirror_flush(sc, bp);
1132                 return;
1133         case BIO_GETATTR:
1134                 if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
1135                         g_mirror_candelete(bp);
1136                         return;
1137                 } else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1138                         g_mirror_kernel_dump(bp);
1139                         return;
1140                 }
1141                 /* FALLTHROUGH */
1142         default:
1143                 g_io_deliver(bp, EOPNOTSUPP);
1144                 return;
1145         }
1146         mtx_lock(&sc->sc_queue_mtx);
1147         bioq_insert_tail(&sc->sc_queue, bp);
1148         mtx_unlock(&sc->sc_queue_mtx);
1149         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1150         wakeup(sc);
1151 }
1152
1153 /*
1154  * Return TRUE if the given request is colliding with a in-progress
1155  * synchronization request.
1156  */
1157 static int
1158 g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1159 {
1160         struct g_mirror_disk *disk;
1161         struct bio *sbp;
1162         off_t rstart, rend, sstart, send;
1163         u_int i;
1164
1165         if (sc->sc_sync.ds_ndisks == 0)
1166                 return (0);
1167         rstart = bp->bio_offset;
1168         rend = bp->bio_offset + bp->bio_length;
1169         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1170                 if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1171                         continue;
1172                 for (i = 0; i < g_mirror_syncreqs; i++) {
1173                         sbp = disk->d_sync.ds_bios[i];
1174                         if (sbp == NULL)
1175                                 continue;
1176                         sstart = sbp->bio_offset;
1177                         send = sbp->bio_offset + sbp->bio_length;
1178                         if (rend > sstart && rstart < send)
1179                                 return (1);
1180                 }
1181         }
1182         return (0);
1183 }
1184
1185 /*
1186  * Return TRUE if the given sync request is colliding with a in-progress regular
1187  * request.
1188  */
1189 static int
1190 g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1191 {
1192         off_t rstart, rend, sstart, send;
1193         struct bio *bp;
1194
1195         if (sc->sc_sync.ds_ndisks == 0)
1196                 return (0);
1197         sstart = sbp->bio_offset;
1198         send = sbp->bio_offset + sbp->bio_length;
1199         TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1200                 rstart = bp->bio_offset;
1201                 rend = bp->bio_offset + bp->bio_length;
1202                 if (rend > sstart && rstart < send)
1203                         return (1);
1204         }
1205         return (0);
1206 }
1207
1208 /*
1209  * Puts request onto delayed queue.
1210  */
1211 static void
1212 g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1213 {
1214
1215         G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1216         bioq_insert_head(&sc->sc_regular_delayed, bp);
1217 }
1218
1219 /*
1220  * Puts synchronization request onto delayed queue.
1221  */
1222 static void
1223 g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1224 {
1225
1226         G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1227         bioq_insert_tail(&sc->sc_sync_delayed, bp);
1228 }
1229
1230 /*
1231  * Releases delayed regular requests which don't collide anymore with sync
1232  * requests.
1233  */
1234 static void
1235 g_mirror_regular_release(struct g_mirror_softc *sc)
1236 {
1237         struct bio *bp, *bp2;
1238
1239         TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1240                 if (g_mirror_sync_collision(sc, bp))
1241                         continue;
1242                 bioq_remove(&sc->sc_regular_delayed, bp);
1243                 G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1244                 mtx_lock(&sc->sc_queue_mtx);
1245                 bioq_insert_head(&sc->sc_queue, bp);
1246 #if 0
1247                 /*
1248                  * wakeup() is not needed, because this function is called from
1249                  * the worker thread.
1250                  */
1251                 wakeup(&sc->sc_queue);
1252 #endif
1253                 mtx_unlock(&sc->sc_queue_mtx);
1254         }
1255 }
1256
1257 /*
1258  * Releases delayed sync requests which don't collide anymore with regular
1259  * requests.
1260  */
1261 static void
1262 g_mirror_sync_release(struct g_mirror_softc *sc)
1263 {
1264         struct bio *bp, *bp2;
1265
1266         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1267                 if (g_mirror_regular_collision(sc, bp))
1268                         continue;
1269                 bioq_remove(&sc->sc_sync_delayed, bp);
1270                 G_MIRROR_LOGREQ(2, bp,
1271                     "Releasing delayed synchronization request.");
1272                 g_io_request(bp, bp->bio_from);
1273         }
1274 }
1275
1276 /*
1277  * Handle synchronization requests.
1278  * Every synchronization request is two-steps process: first, READ request is
1279  * send to active provider and then WRITE request (with read data) to the provider
1280  * being synchronized. When WRITE is finished, new synchronization request is
1281  * send.
1282  */
1283 static void
1284 g_mirror_sync_request(struct bio *bp)
1285 {
1286         struct g_mirror_softc *sc;
1287         struct g_mirror_disk *disk;
1288
1289         bp->bio_from->index--;
1290         sc = bp->bio_from->geom->softc;
1291         disk = bp->bio_from->private;
1292         if (disk == NULL) {
1293                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1294                 g_topology_lock();
1295                 g_mirror_kill_consumer(sc, bp->bio_from);
1296                 g_topology_unlock();
1297                 free(bp->bio_data, M_MIRROR);
1298                 g_destroy_bio(bp);
1299                 sx_xlock(&sc->sc_lock);
1300                 return;
1301         }
1302
1303         /*
1304          * Synchronization request.
1305          */
1306         switch (bp->bio_cmd) {
1307         case BIO_READ:
1308             {
1309                 struct g_consumer *cp;
1310
1311                 if (bp->bio_error != 0) {
1312                         G_MIRROR_LOGREQ(0, bp,
1313                             "Synchronization request failed (error=%d).",
1314                             bp->bio_error);
1315                         g_destroy_bio(bp);
1316                         return;
1317                 }
1318                 G_MIRROR_LOGREQ(3, bp,
1319                     "Synchronization request half-finished.");
1320                 bp->bio_cmd = BIO_WRITE;
1321                 bp->bio_cflags = 0;
1322                 cp = disk->d_consumer;
1323                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1324                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1325                     cp->acr, cp->acw, cp->ace));
1326                 cp->index++;
1327                 g_io_request(bp, cp);
1328                 return;
1329             }
1330         case BIO_WRITE:
1331             {
1332                 struct g_mirror_disk_sync *sync;
1333                 off_t offset;
1334                 void *data;
1335                 int i;
1336
1337                 if (bp->bio_error != 0) {
1338                         G_MIRROR_LOGREQ(0, bp,
1339                             "Synchronization request failed (error=%d).",
1340                             bp->bio_error);
1341                         g_destroy_bio(bp);
1342                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1343                         g_mirror_event_send(disk,
1344                             G_MIRROR_DISK_STATE_DISCONNECTED,
1345                             G_MIRROR_EVENT_DONTWAIT);
1346                         return;
1347                 }
1348                 G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1349                 sync = &disk->d_sync;
1350                 if (sync->ds_offset >= sc->sc_mediasize ||
1351                     sync->ds_consumer == NULL ||
1352                     (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1353                         /* Don't send more synchronization requests. */
1354                         sync->ds_inflight--;
1355                         if (sync->ds_bios != NULL) {
1356                                 i = (int)(uintptr_t)bp->bio_caller1;
1357                                 sync->ds_bios[i] = NULL;
1358                         }
1359                         free(bp->bio_data, M_MIRROR);
1360                         g_destroy_bio(bp);
1361                         if (sync->ds_inflight > 0)
1362                                 return;
1363                         if (sync->ds_consumer == NULL ||
1364                             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1365                                 return;
1366                         }
1367                         /* Disk up-to-date, activate it. */
1368                         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1369                             G_MIRROR_EVENT_DONTWAIT);
1370                         return;
1371                 }
1372
1373                 /* Send next synchronization request. */
1374                 data = bp->bio_data;
1375                 g_reset_bio(bp);
1376                 bp->bio_cmd = BIO_READ;
1377                 bp->bio_offset = sync->ds_offset;
1378                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1379                 sync->ds_offset += bp->bio_length;
1380                 bp->bio_done = g_mirror_sync_done;
1381                 bp->bio_data = data;
1382                 bp->bio_from = sync->ds_consumer;
1383                 bp->bio_to = sc->sc_provider;
1384                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1385                 sync->ds_consumer->index++;
1386                 /*
1387                  * Delay the request if it is colliding with a regular request.
1388                  */
1389                 if (g_mirror_regular_collision(sc, bp))
1390                         g_mirror_sync_delay(sc, bp);
1391                 else
1392                         g_io_request(bp, sync->ds_consumer);
1393
1394                 /* Release delayed requests if possible. */
1395                 g_mirror_regular_release(sc);
1396
1397                 /* Find the smallest offset */
1398                 offset = sc->sc_mediasize;
1399                 for (i = 0; i < g_mirror_syncreqs; i++) {
1400                         bp = sync->ds_bios[i];
1401                         if (bp->bio_offset < offset)
1402                                 offset = bp->bio_offset;
1403                 }
1404                 if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
1405                         /* Update offset_done on every 100 blocks. */
1406                         sync->ds_offset_done = offset;
1407                         g_mirror_update_metadata(disk);
1408                 }
1409                 return;
1410             }
1411         default:
1412                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1413                     bp->bio_cmd, sc->sc_name));
1414                 break;
1415         }
1416 }
1417
1418 static void
1419 g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1420 {
1421         struct g_mirror_disk *disk;
1422         struct g_consumer *cp;
1423         struct bio *cbp;
1424
1425         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1426                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1427                         break;
1428         }
1429         if (disk == NULL) {
1430                 if (bp->bio_error == 0)
1431                         bp->bio_error = ENXIO;
1432                 g_io_deliver(bp, bp->bio_error);
1433                 return;
1434         }
1435         cbp = g_clone_bio(bp);
1436         if (cbp == NULL) {
1437                 if (bp->bio_error == 0)
1438                         bp->bio_error = ENOMEM;
1439                 g_io_deliver(bp, bp->bio_error);
1440                 return;
1441         }
1442         /*
1443          * Fill in the component buf structure.
1444          */
1445         cp = disk->d_consumer;
1446         cbp->bio_done = g_mirror_done;
1447         cbp->bio_to = cp->provider;
1448         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1449         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1450             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1451             cp->acw, cp->ace));
1452         cp->index++;
1453         g_io_request(cbp, cp);
1454 }
1455
1456 static void
1457 g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1458 {
1459         struct g_mirror_disk *disk;
1460         struct g_consumer *cp;
1461         struct bio *cbp;
1462
1463         disk = g_mirror_get_disk(sc);
1464         if (disk == NULL) {
1465                 if (bp->bio_error == 0)
1466                         bp->bio_error = ENXIO;
1467                 g_io_deliver(bp, bp->bio_error);
1468                 return;
1469         }
1470         cbp = g_clone_bio(bp);
1471         if (cbp == NULL) {
1472                 if (bp->bio_error == 0)
1473                         bp->bio_error = ENOMEM;
1474                 g_io_deliver(bp, bp->bio_error);
1475                 return;
1476         }
1477         /*
1478          * Fill in the component buf structure.
1479          */
1480         cp = disk->d_consumer;
1481         cbp->bio_done = g_mirror_done;
1482         cbp->bio_to = cp->provider;
1483         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1484         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1485             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1486             cp->acw, cp->ace));
1487         cp->index++;
1488         g_io_request(cbp, cp);
1489 }
1490
1491 #define TRACK_SIZE  (1 * 1024 * 1024)
1492 #define LOAD_SCALE      256
1493 #define ABS(x)          (((x) >= 0) ? (x) : (-(x)))
1494
1495 static void
1496 g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1497 {
1498         struct g_mirror_disk *disk, *dp;
1499         struct g_consumer *cp;
1500         struct bio *cbp;
1501         int prio, best;
1502
1503         /* Find a disk with the smallest load. */
1504         disk = NULL;
1505         best = INT_MAX;
1506         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1507                 if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1508                         continue;
1509                 prio = dp->load;
1510                 /* If disk head is precisely in position - highly prefer it. */
1511                 if (dp->d_last_offset == bp->bio_offset)
1512                         prio -= 2 * LOAD_SCALE;
1513                 else
1514                 /* If disk head is close to position - prefer it. */
1515                 if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1516                         prio -= 1 * LOAD_SCALE;
1517                 if (prio <= best) {
1518                         disk = dp;
1519                         best = prio;
1520                 }
1521         }
1522         KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1523         cbp = g_clone_bio(bp);
1524         if (cbp == NULL) {
1525                 if (bp->bio_error == 0)
1526                         bp->bio_error = ENOMEM;
1527                 g_io_deliver(bp, bp->bio_error);
1528                 return;
1529         }
1530         /*
1531          * Fill in the component buf structure.
1532          */
1533         cp = disk->d_consumer;
1534         cbp->bio_done = g_mirror_done;
1535         cbp->bio_to = cp->provider;
1536         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1537         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1538             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1539             cp->acw, cp->ace));
1540         cp->index++;
1541         /* Remember last head position */
1542         disk->d_last_offset = bp->bio_offset + bp->bio_length;
1543         /* Update loads. */
1544         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1545                 dp->load = (dp->d_consumer->index * LOAD_SCALE +
1546                     dp->load * 7) / 8;
1547         }
1548         g_io_request(cbp, cp);
1549 }
1550
1551 static void
1552 g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1553 {
1554         struct bio_queue_head queue;
1555         struct g_mirror_disk *disk;
1556         struct g_consumer *cp;
1557         struct bio *cbp;
1558         off_t left, mod, offset, slice;
1559         u_char *data;
1560         u_int ndisks;
1561
1562         if (bp->bio_length <= sc->sc_slice) {
1563                 g_mirror_request_round_robin(sc, bp);
1564                 return;
1565         }
1566         ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1567         slice = bp->bio_length / ndisks;
1568         mod = slice % sc->sc_provider->sectorsize;
1569         if (mod != 0)
1570                 slice += sc->sc_provider->sectorsize - mod;
1571         /*
1572          * Allocate all bios before sending any request, so we can
1573          * return ENOMEM in nice and clean way.
1574          */
1575         left = bp->bio_length;
1576         offset = bp->bio_offset;
1577         data = bp->bio_data;
1578         bioq_init(&queue);
1579         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1580                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1581                         continue;
1582                 cbp = g_clone_bio(bp);
1583                 if (cbp == NULL) {
1584                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1585                                 g_destroy_bio(cbp);
1586                         if (bp->bio_error == 0)
1587                                 bp->bio_error = ENOMEM;
1588                         g_io_deliver(bp, bp->bio_error);
1589                         return;
1590                 }
1591                 bioq_insert_tail(&queue, cbp);
1592                 cbp->bio_done = g_mirror_done;
1593                 cbp->bio_caller1 = disk;
1594                 cbp->bio_to = disk->d_consumer->provider;
1595                 cbp->bio_offset = offset;
1596                 cbp->bio_data = data;
1597                 cbp->bio_length = MIN(left, slice);
1598                 left -= cbp->bio_length;
1599                 if (left == 0)
1600                         break;
1601                 offset += cbp->bio_length;
1602                 data += cbp->bio_length;
1603         }
1604         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1605                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1606                 disk = cbp->bio_caller1;
1607                 cbp->bio_caller1 = NULL;
1608                 cp = disk->d_consumer;
1609                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1610                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1611                     cp->acr, cp->acw, cp->ace));
1612                 disk->d_consumer->index++;
1613                 g_io_request(cbp, disk->d_consumer);
1614         }
1615 }
1616
1617 static void
1618 g_mirror_register_request(struct bio *bp)
1619 {
1620         struct g_mirror_softc *sc;
1621
1622         sc = bp->bio_to->geom->softc;
1623         switch (bp->bio_cmd) {
1624         case BIO_READ:
1625                 switch (sc->sc_balance) {
1626                 case G_MIRROR_BALANCE_LOAD:
1627                         g_mirror_request_load(sc, bp);
1628                         break;
1629                 case G_MIRROR_BALANCE_PREFER:
1630                         g_mirror_request_prefer(sc, bp);
1631                         break;
1632                 case G_MIRROR_BALANCE_ROUND_ROBIN:
1633                         g_mirror_request_round_robin(sc, bp);
1634                         break;
1635                 case G_MIRROR_BALANCE_SPLIT:
1636                         g_mirror_request_split(sc, bp);
1637                         break;
1638                 }
1639                 return;
1640         case BIO_WRITE:
1641         case BIO_DELETE:
1642             {
1643                 struct g_mirror_disk *disk;
1644                 struct g_mirror_disk_sync *sync;
1645                 struct bio_queue_head queue;
1646                 struct g_consumer *cp;
1647                 struct bio *cbp;
1648
1649                 /*
1650                  * Delay the request if it is colliding with a synchronization
1651                  * request.
1652                  */
1653                 if (g_mirror_sync_collision(sc, bp)) {
1654                         g_mirror_regular_delay(sc, bp);
1655                         return;
1656                 }
1657
1658                 if (sc->sc_idle)
1659                         g_mirror_unidle(sc);
1660                 else
1661                         sc->sc_last_write = time_uptime;
1662
1663                 /*
1664                  * Allocate all bios before sending any request, so we can
1665                  * return ENOMEM in nice and clean way.
1666                  */
1667                 bioq_init(&queue);
1668                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1669                         sync = &disk->d_sync;
1670                         switch (disk->d_state) {
1671                         case G_MIRROR_DISK_STATE_ACTIVE:
1672                                 break;
1673                         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1674                                 if (bp->bio_offset >= sync->ds_offset)
1675                                         continue;
1676                                 break;
1677                         default:
1678                                 continue;
1679                         }
1680                         if (bp->bio_cmd == BIO_DELETE &&
1681                             (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1682                                 continue;
1683                         cbp = g_clone_bio(bp);
1684                         if (cbp == NULL) {
1685                                 while ((cbp = bioq_takefirst(&queue)) != NULL)
1686                                         g_destroy_bio(cbp);
1687                                 if (bp->bio_error == 0)
1688                                         bp->bio_error = ENOMEM;
1689                                 g_io_deliver(bp, bp->bio_error);
1690                                 return;
1691                         }
1692                         bioq_insert_tail(&queue, cbp);
1693                         cbp->bio_done = g_mirror_done;
1694                         cp = disk->d_consumer;
1695                         cbp->bio_caller1 = cp;
1696                         cbp->bio_to = cp->provider;
1697                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1698                             ("Consumer %s not opened (r%dw%de%d).",
1699                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1700                 }
1701                 if (bioq_first(&queue) == NULL) {
1702                         g_io_deliver(bp, EOPNOTSUPP);
1703                         return;
1704                 }
1705                 while ((cbp = bioq_takefirst(&queue)) != NULL) {
1706                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1707                         cp = cbp->bio_caller1;
1708                         cbp->bio_caller1 = NULL;
1709                         cp->index++;
1710                         sc->sc_writes++;
1711                         g_io_request(cbp, cp);
1712                 }
1713                 /*
1714                  * Put request onto inflight queue, so we can check if new
1715                  * synchronization requests don't collide with it.
1716                  */
1717                 bioq_insert_tail(&sc->sc_inflight, bp);
1718                 /*
1719                  * Bump syncid on first write.
1720                  */
1721                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1722                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1723                         g_mirror_bump_syncid(sc);
1724                 }
1725                 return;
1726             }
1727         default:
1728                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1729                     bp->bio_cmd, sc->sc_name));
1730                 break;
1731         }
1732 }
1733
1734 static int
1735 g_mirror_can_destroy(struct g_mirror_softc *sc)
1736 {
1737         struct g_geom *gp;
1738         struct g_consumer *cp;
1739
1740         g_topology_assert();
1741         gp = sc->sc_geom;
1742         if (gp->softc == NULL)
1743                 return (1);
1744         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1745                 return (0);
1746         LIST_FOREACH(cp, &gp->consumer, consumer) {
1747                 if (g_mirror_is_busy(sc, cp))
1748                         return (0);
1749         }
1750         gp = sc->sc_sync.ds_geom;
1751         LIST_FOREACH(cp, &gp->consumer, consumer) {
1752                 if (g_mirror_is_busy(sc, cp))
1753                         return (0);
1754         }
1755         G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1756             sc->sc_name);
1757         return (1);
1758 }
1759
1760 static int
1761 g_mirror_try_destroy(struct g_mirror_softc *sc)
1762 {
1763
1764         if (sc->sc_rootmount != NULL) {
1765                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1766                     sc->sc_rootmount);
1767                 root_mount_rel(sc->sc_rootmount);
1768                 sc->sc_rootmount = NULL;
1769         }
1770         g_topology_lock();
1771         if (!g_mirror_can_destroy(sc)) {
1772                 g_topology_unlock();
1773                 return (0);
1774         }
1775         sc->sc_geom->softc = NULL;
1776         sc->sc_sync.ds_geom->softc = NULL;
1777         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
1778                 g_topology_unlock();
1779                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1780                     &sc->sc_worker);
1781                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
1782                 sx_xunlock(&sc->sc_lock);
1783                 wakeup(&sc->sc_worker);
1784                 sc->sc_worker = NULL;
1785         } else {
1786                 g_topology_unlock();
1787                 g_mirror_destroy_device(sc);
1788                 free(sc, M_MIRROR);
1789         }
1790         return (1);
1791 }
1792
1793 /*
1794  * Worker thread.
1795  */
1796 static void
1797 g_mirror_worker(void *arg)
1798 {
1799         struct g_mirror_softc *sc;
1800         struct g_mirror_event *ep;
1801         struct bio *bp;
1802         int timeout;
1803
1804         sc = arg;
1805         thread_lock(curthread);
1806         sched_prio(curthread, PRIBIO);
1807         thread_unlock(curthread);
1808
1809         sx_xlock(&sc->sc_lock);
1810         for (;;) {
1811                 G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1812                 /*
1813                  * First take a look at events.
1814                  * This is important to handle events before any I/O requests.
1815                  */
1816                 ep = g_mirror_event_get(sc);
1817                 if (ep != NULL) {
1818                         g_mirror_event_remove(sc, ep);
1819                         if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1820                                 /* Update only device status. */
1821                                 G_MIRROR_DEBUG(3,
1822                                     "Running event for device %s.",
1823                                     sc->sc_name);
1824                                 ep->e_error = 0;
1825                                 g_mirror_update_device(sc, 1);
1826                         } else {
1827                                 /* Update disk status. */
1828                                 G_MIRROR_DEBUG(3, "Running event for disk %s.",
1829                                      g_mirror_get_diskname(ep->e_disk));
1830                                 ep->e_error = g_mirror_update_disk(ep->e_disk,
1831                                     ep->e_state);
1832                                 if (ep->e_error == 0)
1833                                         g_mirror_update_device(sc, 0);
1834                         }
1835                         if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1836                                 KASSERT(ep->e_error == 0,
1837                                     ("Error cannot be handled."));
1838                                 g_mirror_event_free(ep);
1839                         } else {
1840                                 ep->e_flags |= G_MIRROR_EVENT_DONE;
1841                                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1842                                     ep);
1843                                 mtx_lock(&sc->sc_events_mtx);
1844                                 wakeup(ep);
1845                                 mtx_unlock(&sc->sc_events_mtx);
1846                         }
1847                         if ((sc->sc_flags &
1848                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1849                                 if (g_mirror_try_destroy(sc)) {
1850                                         curthread->td_pflags &= ~TDP_GEOM;
1851                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1852                                         kproc_exit(0);
1853                                 }
1854                         }
1855                         G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1856                         continue;
1857                 }
1858                 /*
1859                  * Check if we can mark array as CLEAN and if we can't take
1860                  * how much seconds should we wait.
1861                  */
1862                 timeout = g_mirror_idle(sc, -1);
1863                 /*
1864                  * Now I/O requests.
1865                  */
1866                 /* Get first request from the queue. */
1867                 mtx_lock(&sc->sc_queue_mtx);
1868                 bp = bioq_takefirst(&sc->sc_queue);
1869                 if (bp == NULL) {
1870                         if ((sc->sc_flags &
1871                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1872                                 mtx_unlock(&sc->sc_queue_mtx);
1873                                 if (g_mirror_try_destroy(sc)) {
1874                                         curthread->td_pflags &= ~TDP_GEOM;
1875                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1876                                         kproc_exit(0);
1877                                 }
1878                                 mtx_lock(&sc->sc_queue_mtx);
1879                         }
1880                         sx_xunlock(&sc->sc_lock);
1881                         /*
1882                          * XXX: We can miss an event here, because an event
1883                          *      can be added without sx-device-lock and without
1884                          *      mtx-queue-lock. Maybe I should just stop using
1885                          *      dedicated mutex for events synchronization and
1886                          *      stick with the queue lock?
1887                          *      The event will hang here until next I/O request
1888                          *      or next event is received.
1889                          */
1890                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1891                             timeout * hz);
1892                         sx_xlock(&sc->sc_lock);
1893                         G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1894                         continue;
1895                 }
1896                 mtx_unlock(&sc->sc_queue_mtx);
1897
1898                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
1899                     (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
1900                         g_mirror_sync_request(bp);      /* READ */
1901                 } else if (bp->bio_to != sc->sc_provider) {
1902                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
1903                                 g_mirror_regular_request(bp);
1904                         else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
1905                                 g_mirror_sync_request(bp);      /* WRITE */
1906                         else {
1907                                 KASSERT(0,
1908                                     ("Invalid request cflags=0x%hx to=%s.",
1909                                     bp->bio_cflags, bp->bio_to->name));
1910                         }
1911                 } else {
1912                         g_mirror_register_request(bp);
1913                 }
1914                 G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
1915         }
1916 }
1917
1918 static void
1919 g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
1920 {
1921
1922         sx_assert(&sc->sc_lock, SX_LOCKED);
1923
1924         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
1925                 return;
1926         if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
1927                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as dirty.",
1928                     g_mirror_get_diskname(disk), sc->sc_name);
1929                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1930         } else if (sc->sc_idle &&
1931             (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
1932                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as clean.",
1933                     g_mirror_get_diskname(disk), sc->sc_name);
1934                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
1935         }
1936 }
1937
1938 static void
1939 g_mirror_sync_start(struct g_mirror_disk *disk)
1940 {
1941         struct g_mirror_softc *sc;
1942         struct g_consumer *cp;
1943         struct bio *bp;
1944         int error, i;
1945
1946         g_topology_assert_not();
1947         sc = disk->d_softc;
1948         sx_assert(&sc->sc_lock, SX_LOCKED);
1949
1950         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
1951             ("Disk %s is not marked for synchronization.",
1952             g_mirror_get_diskname(disk)));
1953         KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1954             ("Device not in RUNNING state (%s, %u).", sc->sc_name,
1955             sc->sc_state));
1956
1957         sx_xunlock(&sc->sc_lock);
1958         g_topology_lock();
1959         cp = g_new_consumer(sc->sc_sync.ds_geom);
1960         cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
1961         error = g_attach(cp, sc->sc_provider);
1962         KASSERT(error == 0,
1963             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
1964         error = g_access(cp, 1, 0, 0);
1965         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
1966         g_topology_unlock();
1967         sx_xlock(&sc->sc_lock);
1968
1969         G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
1970             g_mirror_get_diskname(disk));
1971         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
1972                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1973         KASSERT(disk->d_sync.ds_consumer == NULL,
1974             ("Sync consumer already exists (device=%s, disk=%s).",
1975             sc->sc_name, g_mirror_get_diskname(disk)));
1976
1977         disk->d_sync.ds_consumer = cp;
1978         disk->d_sync.ds_consumer->private = disk;
1979         disk->d_sync.ds_consumer->index = 0;
1980
1981         /*
1982          * Allocate memory for synchronization bios and initialize them.
1983          */
1984         disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
1985             M_MIRROR, M_WAITOK);
1986         for (i = 0; i < g_mirror_syncreqs; i++) {
1987                 bp = g_alloc_bio();
1988                 disk->d_sync.ds_bios[i] = bp;
1989                 bp->bio_parent = NULL;
1990                 bp->bio_cmd = BIO_READ;
1991                 bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
1992                 bp->bio_cflags = 0;
1993                 bp->bio_offset = disk->d_sync.ds_offset;
1994                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1995                 disk->d_sync.ds_offset += bp->bio_length;
1996                 bp->bio_done = g_mirror_sync_done;
1997                 bp->bio_from = disk->d_sync.ds_consumer;
1998                 bp->bio_to = sc->sc_provider;
1999                 bp->bio_caller1 = (void *)(uintptr_t)i;
2000         }
2001
2002         /* Increase the number of disks in SYNCHRONIZING state. */
2003         sc->sc_sync.ds_ndisks++;
2004         /* Set the number of in-flight synchronization requests. */
2005         disk->d_sync.ds_inflight = g_mirror_syncreqs;
2006
2007         /*
2008          * Fire off first synchronization requests.
2009          */
2010         for (i = 0; i < g_mirror_syncreqs; i++) {
2011                 bp = disk->d_sync.ds_bios[i];
2012                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
2013                 disk->d_sync.ds_consumer->index++;
2014                 /*
2015                  * Delay the request if it is colliding with a regular request.
2016                  */
2017                 if (g_mirror_regular_collision(sc, bp))
2018                         g_mirror_sync_delay(sc, bp);
2019                 else
2020                         g_io_request(bp, disk->d_sync.ds_consumer);
2021         }
2022 }
2023
2024 /*
2025  * Stop synchronization process.
2026  * type: 0 - synchronization finished
2027  *       1 - synchronization stopped
2028  */
2029 static void
2030 g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2031 {
2032         struct g_mirror_softc *sc;
2033         struct g_consumer *cp;
2034
2035         g_topology_assert_not();
2036         sc = disk->d_softc;
2037         sx_assert(&sc->sc_lock, SX_LOCKED);
2038
2039         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2040             ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2041             g_mirror_disk_state2str(disk->d_state)));
2042         if (disk->d_sync.ds_consumer == NULL)
2043                 return;
2044
2045         if (type == 0) {
2046                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2047                     sc->sc_name, g_mirror_get_diskname(disk));
2048         } else /* if (type == 1) */ {
2049                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2050                     sc->sc_name, g_mirror_get_diskname(disk));
2051         }
2052         free(disk->d_sync.ds_bios, M_MIRROR);
2053         disk->d_sync.ds_bios = NULL;
2054         cp = disk->d_sync.ds_consumer;
2055         disk->d_sync.ds_consumer = NULL;
2056         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2057         sc->sc_sync.ds_ndisks--;
2058         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2059         g_topology_lock();
2060         g_mirror_kill_consumer(sc, cp);
2061         g_topology_unlock();
2062         sx_xlock(&sc->sc_lock);
2063 }
2064
2065 static void
2066 g_mirror_launch_provider(struct g_mirror_softc *sc)
2067 {
2068         struct g_mirror_disk *disk;
2069         struct g_provider *pp, *dp;
2070
2071         sx_assert(&sc->sc_lock, SX_LOCKED);
2072
2073         g_topology_lock();
2074         pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2075         pp->flags |= G_PF_DIRECT_RECEIVE;
2076         pp->mediasize = sc->sc_mediasize;
2077         pp->sectorsize = sc->sc_sectorsize;
2078         pp->stripesize = 0;
2079         pp->stripeoffset = 0;
2080
2081         /* Splitting of unmapped BIO's could work but isn't implemented now */
2082         if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2083                 pp->flags |= G_PF_ACCEPT_UNMAPPED;
2084
2085         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2086                 if (disk->d_consumer && disk->d_consumer->provider) {
2087                         dp = disk->d_consumer->provider;
2088                         if (dp->stripesize > pp->stripesize) {
2089                                 pp->stripesize = dp->stripesize;
2090                                 pp->stripeoffset = dp->stripeoffset;
2091                         }
2092                         /* A provider underneath us doesn't support unmapped */
2093                         if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2094                                 G_MIRROR_DEBUG(0, "Cancelling unmapped "
2095                                     "because of %s.", dp->name);
2096                                 pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2097                         }
2098                 }
2099         }
2100         sc->sc_provider = pp;
2101         g_error_provider(pp, 0);
2102         g_topology_unlock();
2103         G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2104             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2105         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2106                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2107                         g_mirror_sync_start(disk);
2108         }
2109 }
2110
2111 static void
2112 g_mirror_destroy_provider(struct g_mirror_softc *sc)
2113 {
2114         struct g_mirror_disk *disk;
2115         struct bio *bp;
2116
2117         g_topology_assert_not();
2118         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2119             sc->sc_name));
2120
2121         g_topology_lock();
2122         g_error_provider(sc->sc_provider, ENXIO);
2123         mtx_lock(&sc->sc_queue_mtx);
2124         while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL) {
2125                 /*
2126                  * Abort any pending I/O that wasn't generated by us.
2127                  * Synchronization requests and requests destined for individual
2128                  * mirror components can be destroyed immediately.
2129                  */
2130                 if (bp->bio_to == sc->sc_provider &&
2131                     bp->bio_from->geom != sc->sc_sync.ds_geom) {
2132                         g_io_deliver(bp, ENXIO);
2133                 } else {
2134                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
2135                                 free(bp->bio_data, M_MIRROR);
2136                         g_destroy_bio(bp);
2137                 }
2138         }
2139         mtx_unlock(&sc->sc_queue_mtx);
2140         G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2141             sc->sc_provider->name);
2142         sc->sc_provider->flags |= G_PF_WITHER;
2143         g_orphan_provider(sc->sc_provider, ENXIO);
2144         g_topology_unlock();
2145         sc->sc_provider = NULL;
2146         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2147                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2148                         g_mirror_sync_stop(disk, 1);
2149         }
2150 }
2151
2152 static void
2153 g_mirror_go(void *arg)
2154 {
2155         struct g_mirror_softc *sc;
2156
2157         sc = arg;
2158         G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2159         g_mirror_event_send(sc, 0,
2160             G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2161 }
2162
2163 static u_int
2164 g_mirror_determine_state(struct g_mirror_disk *disk)
2165 {
2166         struct g_mirror_softc *sc;
2167         u_int state;
2168
2169         sc = disk->d_softc;
2170         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2171                 if ((disk->d_flags &
2172                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2173                         /* Disk does not need synchronization. */
2174                         state = G_MIRROR_DISK_STATE_ACTIVE;
2175                 } else {
2176                         if ((sc->sc_flags &
2177                              G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2178                             (disk->d_flags &
2179                              G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2180                                 /*
2181                                  * We can start synchronization from
2182                                  * the stored offset.
2183                                  */
2184                                 state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2185                         } else {
2186                                 state = G_MIRROR_DISK_STATE_STALE;
2187                         }
2188                 }
2189         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2190                 /*
2191                  * Reset all synchronization data for this disk,
2192                  * because if it even was synchronized, it was
2193                  * synchronized to disks with different syncid.
2194                  */
2195                 disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2196                 disk->d_sync.ds_offset = 0;
2197                 disk->d_sync.ds_offset_done = 0;
2198                 disk->d_sync.ds_syncid = sc->sc_syncid;
2199                 if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2200                     (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2201                         state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2202                 } else {
2203                         state = G_MIRROR_DISK_STATE_STALE;
2204                 }
2205         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2206                 /*
2207                  * Not good, NOT GOOD!
2208                  * It means that mirror was started on stale disks
2209                  * and more fresh disk just arrive.
2210                  * If there were writes, mirror is broken, sorry.
2211                  * I think the best choice here is don't touch
2212                  * this disk and inform the user loudly.
2213                  */
2214                 G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2215                     "disk (%s) arrives!! It will not be connected to the "
2216                     "running device.", sc->sc_name,
2217                     g_mirror_get_diskname(disk));
2218                 g_mirror_destroy_disk(disk);
2219                 state = G_MIRROR_DISK_STATE_NONE;
2220                 /* Return immediately, because disk was destroyed. */
2221                 return (state);
2222         }
2223         G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2224             g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2225         return (state);
2226 }
2227
2228 /*
2229  * Update device state.
2230  */
2231 static void
2232 g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
2233 {
2234         struct g_mirror_disk *disk;
2235         u_int state;
2236
2237         sx_assert(&sc->sc_lock, SX_XLOCKED);
2238
2239         switch (sc->sc_state) {
2240         case G_MIRROR_DEVICE_STATE_STARTING:
2241             {
2242                 struct g_mirror_disk *pdisk, *tdisk;
2243                 u_int dirty, ndisks, genid, syncid;
2244
2245                 KASSERT(sc->sc_provider == NULL,
2246                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2247                 /*
2248                  * Are we ready? We are, if all disks are connected or
2249                  * if we have any disks and 'force' is true.
2250                  */
2251                 ndisks = g_mirror_ndisks(sc, -1);
2252                 if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
2253                         ;
2254                 } else if (ndisks == 0) {
2255                         /*
2256                          * Disks went down in starting phase, so destroy
2257                          * device.
2258                          */
2259                         callout_drain(&sc->sc_callout);
2260                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2261                         G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2262                             sc->sc_rootmount);
2263                         root_mount_rel(sc->sc_rootmount);
2264                         sc->sc_rootmount = NULL;
2265                         return;
2266                 } else {
2267                         return;
2268                 }
2269
2270                 /*
2271                  * Activate all disks with the biggest syncid.
2272                  */
2273                 if (force) {
2274                         /*
2275                          * If 'force' is true, we have been called due to
2276                          * timeout, so don't bother canceling timeout.
2277                          */
2278                         ndisks = 0;
2279                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2280                                 if ((disk->d_flags &
2281                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2282                                         ndisks++;
2283                                 }
2284                         }
2285                         if (ndisks == 0) {
2286                                 /* No valid disks found, destroy device. */
2287                                 sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2288                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2289                                     __LINE__, sc->sc_rootmount);
2290                                 root_mount_rel(sc->sc_rootmount);
2291                                 sc->sc_rootmount = NULL;
2292                                 return;
2293                         }
2294                 } else {
2295                         /* Cancel timeout. */
2296                         callout_drain(&sc->sc_callout);
2297                 }
2298
2299                 /*
2300                  * Find the biggest genid.
2301                  */
2302                 genid = 0;
2303                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2304                         if (disk->d_genid > genid)
2305                                 genid = disk->d_genid;
2306                 }
2307                 sc->sc_genid = genid;
2308                 /*
2309                  * Remove all disks without the biggest genid.
2310                  */
2311                 LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2312                         if (disk->d_genid < genid) {
2313                                 G_MIRROR_DEBUG(0,
2314                                     "Component %s (device %s) broken, skipping.",
2315                                     g_mirror_get_diskname(disk), sc->sc_name);
2316                                 g_mirror_destroy_disk(disk);
2317                         }
2318                 }
2319
2320                 /*
2321                  * Find the biggest syncid.
2322                  */
2323                 syncid = 0;
2324                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2325                         if (disk->d_sync.ds_syncid > syncid)
2326                                 syncid = disk->d_sync.ds_syncid;
2327                 }
2328
2329                 /*
2330                  * Here we need to look for dirty disks and if all disks
2331                  * with the biggest syncid are dirty, we have to choose
2332                  * one with the biggest priority and rebuild the rest.
2333                  */
2334                 /*
2335                  * Find the number of dirty disks with the biggest syncid.
2336                  * Find the number of disks with the biggest syncid.
2337                  * While here, find a disk with the biggest priority.
2338                  */
2339                 dirty = ndisks = 0;
2340                 pdisk = NULL;
2341                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2342                         if (disk->d_sync.ds_syncid != syncid)
2343                                 continue;
2344                         if ((disk->d_flags &
2345                             G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2346                                 continue;
2347                         }
2348                         ndisks++;
2349                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2350                                 dirty++;
2351                                 if (pdisk == NULL ||
2352                                     pdisk->d_priority < disk->d_priority) {
2353                                         pdisk = disk;
2354                                 }
2355                         }
2356                 }
2357                 if (dirty == 0) {
2358                         /* No dirty disks at all, great. */
2359                 } else if (dirty == ndisks) {
2360                         /*
2361                          * Force synchronization for all dirty disks except one
2362                          * with the biggest priority.
2363                          */
2364                         KASSERT(pdisk != NULL, ("pdisk == NULL"));
2365                         G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2366                             "master disk for synchronization.",
2367                             g_mirror_get_diskname(pdisk), sc->sc_name);
2368                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2369                                 if (disk->d_sync.ds_syncid != syncid)
2370                                         continue;
2371                                 if ((disk->d_flags &
2372                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2373                                         continue;
2374                                 }
2375                                 KASSERT((disk->d_flags &
2376                                     G_MIRROR_DISK_FLAG_DIRTY) != 0,
2377                                     ("Disk %s isn't marked as dirty.",
2378                                     g_mirror_get_diskname(disk)));
2379                                 /* Skip the disk with the biggest priority. */
2380                                 if (disk == pdisk)
2381                                         continue;
2382                                 disk->d_sync.ds_syncid = 0;
2383                         }
2384                 } else if (dirty < ndisks) {
2385                         /*
2386                          * Force synchronization for all dirty disks.
2387                          * We have some non-dirty disks.
2388                          */
2389                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2390                                 if (disk->d_sync.ds_syncid != syncid)
2391                                         continue;
2392                                 if ((disk->d_flags &
2393                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2394                                         continue;
2395                                 }
2396                                 if ((disk->d_flags &
2397                                     G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2398                                         continue;
2399                                 }
2400                                 disk->d_sync.ds_syncid = 0;
2401                         }
2402                 }
2403
2404                 /* Reset hint. */
2405                 sc->sc_hint = NULL;
2406                 sc->sc_syncid = syncid;
2407                 if (force) {
2408                         /* Remember to bump syncid on first write. */
2409                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2410                 }
2411                 state = G_MIRROR_DEVICE_STATE_RUNNING;
2412                 G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2413                     sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2414                     g_mirror_device_state2str(state));
2415                 sc->sc_state = state;
2416                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2417                         state = g_mirror_determine_state(disk);
2418                         g_mirror_event_send(disk, state,
2419                             G_MIRROR_EVENT_DONTWAIT);
2420                         if (state == G_MIRROR_DISK_STATE_STALE)
2421                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2422                 }
2423                 break;
2424             }
2425         case G_MIRROR_DEVICE_STATE_RUNNING:
2426                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2427                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2428                         /*
2429                          * No active disks or no disks at all,
2430                          * so destroy device.
2431                          */
2432                         if (sc->sc_provider != NULL)
2433                                 g_mirror_destroy_provider(sc);
2434                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2435                         break;
2436                 } else if (g_mirror_ndisks(sc,
2437                     G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2438                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2439                         /*
2440                          * We have active disks, launch provider if it doesn't
2441                          * exist.
2442                          */
2443                         if (sc->sc_provider == NULL)
2444                                 g_mirror_launch_provider(sc);
2445                         if (sc->sc_rootmount != NULL) {
2446                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2447                                     __LINE__, sc->sc_rootmount);
2448                                 root_mount_rel(sc->sc_rootmount);
2449                                 sc->sc_rootmount = NULL;
2450                         }
2451                 }
2452                 /*
2453                  * Genid should be bumped immediately, so do it here.
2454                  */
2455                 if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2456                         sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2457                         g_mirror_bump_genid(sc);
2458                 }
2459                 break;
2460         default:
2461                 KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2462                     sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2463                 break;
2464         }
2465 }
2466
2467 /*
2468  * Update disk state and device state if needed.
2469  */
2470 #define DISK_STATE_CHANGED()    G_MIRROR_DEBUG(1,                       \
2471         "Disk %s state changed from %s to %s (device %s).",             \
2472         g_mirror_get_diskname(disk),                                    \
2473         g_mirror_disk_state2str(disk->d_state),                         \
2474         g_mirror_disk_state2str(state), sc->sc_name)
2475 static int
2476 g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2477 {
2478         struct g_mirror_softc *sc;
2479
2480         sc = disk->d_softc;
2481         sx_assert(&sc->sc_lock, SX_XLOCKED);
2482
2483 again:
2484         G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2485             g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2486             g_mirror_disk_state2str(state));
2487         switch (state) {
2488         case G_MIRROR_DISK_STATE_NEW:
2489                 /*
2490                  * Possible scenarios:
2491                  * 1. New disk arrive.
2492                  */
2493                 /* Previous state should be NONE. */
2494                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2495                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2496                     g_mirror_disk_state2str(disk->d_state)));
2497                 DISK_STATE_CHANGED();
2498
2499                 disk->d_state = state;
2500                 if (LIST_EMPTY(&sc->sc_disks))
2501                         LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2502                 else {
2503                         struct g_mirror_disk *dp;
2504
2505                         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2506                                 if (disk->d_priority >= dp->d_priority) {
2507                                         LIST_INSERT_BEFORE(dp, disk, d_next);
2508                                         dp = NULL;
2509                                         break;
2510                                 }
2511                                 if (LIST_NEXT(dp, d_next) == NULL)
2512                                         break;
2513                         }
2514                         if (dp != NULL)
2515                                 LIST_INSERT_AFTER(dp, disk, d_next);
2516                 }
2517                 G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2518                     sc->sc_name, g_mirror_get_diskname(disk));
2519                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2520                         break;
2521                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2522                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2523                     g_mirror_device_state2str(sc->sc_state),
2524                     g_mirror_get_diskname(disk),
2525                     g_mirror_disk_state2str(disk->d_state)));
2526                 state = g_mirror_determine_state(disk);
2527                 if (state != G_MIRROR_DISK_STATE_NONE)
2528                         goto again;
2529                 break;
2530         case G_MIRROR_DISK_STATE_ACTIVE:
2531                 /*
2532                  * Possible scenarios:
2533                  * 1. New disk does not need synchronization.
2534                  * 2. Synchronization process finished successfully.
2535                  */
2536                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2537                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2538                     g_mirror_device_state2str(sc->sc_state),
2539                     g_mirror_get_diskname(disk),
2540                     g_mirror_disk_state2str(disk->d_state)));
2541                 /* Previous state should be NEW or SYNCHRONIZING. */
2542                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2543                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2544                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2545                     g_mirror_disk_state2str(disk->d_state)));
2546                 DISK_STATE_CHANGED();
2547
2548                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2549                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2550                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2551                         g_mirror_sync_stop(disk, 0);
2552                 }
2553                 disk->d_state = state;
2554                 disk->d_sync.ds_offset = 0;
2555                 disk->d_sync.ds_offset_done = 0;
2556                 g_mirror_update_idle(sc, disk);
2557                 g_mirror_update_metadata(disk);
2558                 G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2559                     sc->sc_name, g_mirror_get_diskname(disk));
2560                 break;
2561         case G_MIRROR_DISK_STATE_STALE:
2562                 /*
2563                  * Possible scenarios:
2564                  * 1. Stale disk was connected.
2565                  */
2566                 /* Previous state should be NEW. */
2567                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2568                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2569                     g_mirror_disk_state2str(disk->d_state)));
2570                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2571                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2572                     g_mirror_device_state2str(sc->sc_state),
2573                     g_mirror_get_diskname(disk),
2574                     g_mirror_disk_state2str(disk->d_state)));
2575                 /*
2576                  * STALE state is only possible if device is marked
2577                  * NOAUTOSYNC.
2578                  */
2579                 KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2580                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2581                     g_mirror_device_state2str(sc->sc_state),
2582                     g_mirror_get_diskname(disk),
2583                     g_mirror_disk_state2str(disk->d_state)));
2584                 DISK_STATE_CHANGED();
2585
2586                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2587                 disk->d_state = state;
2588                 g_mirror_update_metadata(disk);
2589                 G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2590                     sc->sc_name, g_mirror_get_diskname(disk));
2591                 break;
2592         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2593                 /*
2594                  * Possible scenarios:
2595                  * 1. Disk which needs synchronization was connected.
2596                  */
2597                 /* Previous state should be NEW. */
2598                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2599                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2600                     g_mirror_disk_state2str(disk->d_state)));
2601                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2602                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2603                     g_mirror_device_state2str(sc->sc_state),
2604                     g_mirror_get_diskname(disk),
2605                     g_mirror_disk_state2str(disk->d_state)));
2606                 DISK_STATE_CHANGED();
2607
2608                 if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2609                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2610                 disk->d_state = state;
2611                 if (sc->sc_provider != NULL) {
2612                         g_mirror_sync_start(disk);
2613                         g_mirror_update_metadata(disk);
2614                 }
2615                 break;
2616         case G_MIRROR_DISK_STATE_DISCONNECTED:
2617                 /*
2618                  * Possible scenarios:
2619                  * 1. Device wasn't running yet, but disk disappear.
2620                  * 2. Disk was active and disapppear.
2621                  * 3. Disk disappear during synchronization process.
2622                  */
2623                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2624                         /*
2625                          * Previous state should be ACTIVE, STALE or
2626                          * SYNCHRONIZING.
2627                          */
2628                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2629                             disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2630                             disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2631                             ("Wrong disk state (%s, %s).",
2632                             g_mirror_get_diskname(disk),
2633                             g_mirror_disk_state2str(disk->d_state)));
2634                 } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2635                         /* Previous state should be NEW. */
2636                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2637                             ("Wrong disk state (%s, %s).",
2638                             g_mirror_get_diskname(disk),
2639                             g_mirror_disk_state2str(disk->d_state)));
2640                         /*
2641                          * Reset bumping syncid if disk disappeared in STARTING
2642                          * state.
2643                          */
2644                         if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2645                                 sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2646 #ifdef  INVARIANTS
2647                 } else {
2648                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2649                             sc->sc_name,
2650                             g_mirror_device_state2str(sc->sc_state),
2651                             g_mirror_get_diskname(disk),
2652                             g_mirror_disk_state2str(disk->d_state)));
2653 #endif
2654                 }
2655                 DISK_STATE_CHANGED();
2656                 G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2657                     sc->sc_name, g_mirror_get_diskname(disk));
2658
2659                 g_mirror_destroy_disk(disk);
2660                 break;
2661         case G_MIRROR_DISK_STATE_DESTROY:
2662             {
2663                 int error;
2664
2665                 error = g_mirror_clear_metadata(disk);
2666                 if (error != 0)
2667                         return (error);
2668                 DISK_STATE_CHANGED();
2669                 G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2670                     sc->sc_name, g_mirror_get_diskname(disk));
2671
2672                 g_mirror_destroy_disk(disk);
2673                 sc->sc_ndisks--;
2674                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2675                         g_mirror_update_metadata(disk);
2676                 }
2677                 break;
2678             }
2679         default:
2680                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2681                 break;
2682         }
2683         return (0);
2684 }
2685 #undef  DISK_STATE_CHANGED
2686
2687 int
2688 g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2689 {
2690         struct g_provider *pp;
2691         u_char *buf;
2692         int error;
2693
2694         g_topology_assert();
2695
2696         error = g_access(cp, 1, 0, 0);
2697         if (error != 0)
2698                 return (error);
2699         pp = cp->provider;
2700         g_topology_unlock();
2701         /* Metadata are stored on last sector. */
2702         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2703             &error);
2704         g_topology_lock();
2705         g_access(cp, -1, 0, 0);
2706         if (buf == NULL) {
2707                 G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2708                     cp->provider->name, error);
2709                 return (error);
2710         }
2711
2712         /* Decode metadata. */
2713         error = mirror_metadata_decode(buf, md);
2714         g_free(buf);
2715         if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2716                 return (EINVAL);
2717         if (md->md_version > G_MIRROR_VERSION) {
2718                 G_MIRROR_DEBUG(0,
2719                     "Kernel module is too old to handle metadata from %s.",
2720                     cp->provider->name);
2721                 return (EINVAL);
2722         }
2723         if (error != 0) {
2724                 G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2725                     cp->provider->name);
2726                 return (error);
2727         }
2728
2729         return (0);
2730 }
2731
2732 static int
2733 g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2734     struct g_mirror_metadata *md)
2735 {
2736
2737         if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2738                 G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2739                     pp->name, md->md_did);
2740                 return (EEXIST);
2741         }
2742         if (md->md_all != sc->sc_ndisks) {
2743                 G_MIRROR_DEBUG(1,
2744                     "Invalid '%s' field on disk %s (device %s), skipping.",
2745                     "md_all", pp->name, sc->sc_name);
2746                 return (EINVAL);
2747         }
2748         if (md->md_slice != sc->sc_slice) {
2749                 G_MIRROR_DEBUG(1,
2750                     "Invalid '%s' field on disk %s (device %s), skipping.",
2751                     "md_slice", pp->name, sc->sc_name);
2752                 return (EINVAL);
2753         }
2754         if (md->md_balance != sc->sc_balance) {
2755                 G_MIRROR_DEBUG(1,
2756                     "Invalid '%s' field on disk %s (device %s), skipping.",
2757                     "md_balance", pp->name, sc->sc_name);
2758                 return (EINVAL);
2759         }
2760 #if 0
2761         if (md->md_mediasize != sc->sc_mediasize) {
2762                 G_MIRROR_DEBUG(1,
2763                     "Invalid '%s' field on disk %s (device %s), skipping.",
2764                     "md_mediasize", pp->name, sc->sc_name);
2765                 return (EINVAL);
2766         }
2767 #endif
2768         if (sc->sc_mediasize > pp->mediasize) {
2769                 G_MIRROR_DEBUG(1,
2770                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2771                     sc->sc_name);
2772                 return (EINVAL);
2773         }
2774         if (md->md_sectorsize != sc->sc_sectorsize) {
2775                 G_MIRROR_DEBUG(1,
2776                     "Invalid '%s' field on disk %s (device %s), skipping.",
2777                     "md_sectorsize", pp->name, sc->sc_name);
2778                 return (EINVAL);
2779         }
2780         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2781                 G_MIRROR_DEBUG(1,
2782                     "Invalid sector size of disk %s (device %s), skipping.",
2783                     pp->name, sc->sc_name);
2784                 return (EINVAL);
2785         }
2786         if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2787                 G_MIRROR_DEBUG(1,
2788                     "Invalid device flags on disk %s (device %s), skipping.",
2789                     pp->name, sc->sc_name);
2790                 return (EINVAL);
2791         }
2792         if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2793                 G_MIRROR_DEBUG(1,
2794                     "Invalid disk flags on disk %s (device %s), skipping.",
2795                     pp->name, sc->sc_name);
2796                 return (EINVAL);
2797         }
2798         return (0);
2799 }
2800
2801 int
2802 g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2803     struct g_mirror_metadata *md)
2804 {
2805         struct g_mirror_disk *disk;
2806         int error;
2807
2808         g_topology_assert_not();
2809         G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2810
2811         error = g_mirror_check_metadata(sc, pp, md);
2812         if (error != 0)
2813                 return (error);
2814         if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
2815             md->md_genid < sc->sc_genid) {
2816                 G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2817                     pp->name, sc->sc_name);
2818                 return (EINVAL);
2819         }
2820         disk = g_mirror_init_disk(sc, pp, md, &error);
2821         if (disk == NULL)
2822                 return (error);
2823         error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2824             G_MIRROR_EVENT_WAIT);
2825         if (error != 0)
2826                 return (error);
2827         if (md->md_version < G_MIRROR_VERSION) {
2828                 G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2829                     pp->name, md->md_version, G_MIRROR_VERSION);
2830                 g_mirror_update_metadata(disk);
2831         }
2832         return (0);
2833 }
2834
2835 static void
2836 g_mirror_destroy_delayed(void *arg, int flag)
2837 {
2838         struct g_mirror_softc *sc;
2839         int error;
2840
2841         if (flag == EV_CANCEL) {
2842                 G_MIRROR_DEBUG(1, "Destroying canceled.");
2843                 return;
2844         }
2845         sc = arg;
2846         g_topology_unlock();
2847         sx_xlock(&sc->sc_lock);
2848         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
2849             ("DESTROY flag set on %s.", sc->sc_name));
2850         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
2851             ("DESTROYING flag not set on %s.", sc->sc_name));
2852         G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
2853         error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
2854         if (error != 0) {
2855                 G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
2856                     sc->sc_name, error);
2857                 sx_xunlock(&sc->sc_lock);
2858         }
2859         g_topology_lock();
2860 }
2861
2862 static int
2863 g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
2864 {
2865         struct g_mirror_softc *sc;
2866         int dcr, dcw, dce, error = 0;
2867
2868         g_topology_assert();
2869         G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
2870             acw, ace);
2871
2872         sc = pp->geom->softc;
2873         if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
2874                 return (0);
2875         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
2876
2877         dcr = pp->acr + acr;
2878         dcw = pp->acw + acw;
2879         dce = pp->ace + ace;
2880
2881         g_topology_unlock();
2882         sx_xlock(&sc->sc_lock);
2883         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
2884             LIST_EMPTY(&sc->sc_disks)) {
2885                 if (acr > 0 || acw > 0 || ace > 0)
2886                         error = ENXIO;
2887                 goto end;
2888         }
2889         if (dcw == 0)
2890                 g_mirror_idle(sc, dcw);
2891         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
2892                 if (acr > 0 || acw > 0 || ace > 0) {
2893                         error = ENXIO;
2894                         goto end;
2895                 }
2896                 if (dcr == 0 && dcw == 0 && dce == 0) {
2897                         g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
2898                             sc, NULL);
2899                 }
2900         }
2901 end:
2902         sx_xunlock(&sc->sc_lock);
2903         g_topology_lock();
2904         return (error);
2905 }
2906
2907 static struct g_geom *
2908 g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
2909 {
2910         struct g_mirror_softc *sc;
2911         struct g_geom *gp;
2912         int error, timeout;
2913
2914         g_topology_assert();
2915         G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
2916             md->md_mid);
2917
2918         /* One disk is minimum. */
2919         if (md->md_all < 1)
2920                 return (NULL);
2921         /*
2922          * Action geom.
2923          */
2924         gp = g_new_geomf(mp, "%s", md->md_name);
2925         sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
2926         gp->start = g_mirror_start;
2927         gp->orphan = g_mirror_orphan;
2928         gp->access = g_mirror_access;
2929         gp->dumpconf = g_mirror_dumpconf;
2930
2931         sc->sc_id = md->md_mid;
2932         sc->sc_slice = md->md_slice;
2933         sc->sc_balance = md->md_balance;
2934         sc->sc_mediasize = md->md_mediasize;
2935         sc->sc_sectorsize = md->md_sectorsize;
2936         sc->sc_ndisks = md->md_all;
2937         sc->sc_flags = md->md_mflags;
2938         sc->sc_bump_id = 0;
2939         sc->sc_idle = 1;
2940         sc->sc_last_write = time_uptime;
2941         sc->sc_writes = 0;
2942         sx_init(&sc->sc_lock, "gmirror:lock");
2943         bioq_init(&sc->sc_queue);
2944         mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
2945         bioq_init(&sc->sc_regular_delayed);
2946         bioq_init(&sc->sc_inflight);
2947         bioq_init(&sc->sc_sync_delayed);
2948         LIST_INIT(&sc->sc_disks);
2949         TAILQ_INIT(&sc->sc_events);
2950         mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
2951         callout_init(&sc->sc_callout, 1);
2952         mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
2953         sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
2954         gp->softc = sc;
2955         sc->sc_geom = gp;
2956         sc->sc_provider = NULL;
2957         /*
2958          * Synchronization geom.
2959          */
2960         gp = g_new_geomf(mp, "%s.sync", md->md_name);
2961         gp->softc = sc;
2962         gp->orphan = g_mirror_orphan;
2963         sc->sc_sync.ds_geom = gp;
2964         sc->sc_sync.ds_ndisks = 0;
2965         error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
2966             "g_mirror %s", md->md_name);
2967         if (error != 0) {
2968                 G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
2969                     sc->sc_name);
2970                 g_destroy_geom(sc->sc_sync.ds_geom);
2971                 mtx_destroy(&sc->sc_done_mtx);
2972                 mtx_destroy(&sc->sc_events_mtx);
2973                 mtx_destroy(&sc->sc_queue_mtx);
2974                 sx_destroy(&sc->sc_lock);
2975                 g_destroy_geom(sc->sc_geom);
2976                 free(sc, M_MIRROR);
2977                 return (NULL);
2978         }
2979
2980         G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
2981             sc->sc_name, sc->sc_ndisks, sc->sc_id);
2982
2983         sc->sc_rootmount = root_mount_hold("GMIRROR");
2984         G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
2985         /*
2986          * Run timeout.
2987          */
2988         timeout = g_mirror_timeout * hz;
2989         callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
2990         return (sc->sc_geom);
2991 }
2992
2993 int
2994 g_mirror_destroy(struct g_mirror_softc *sc, int how)
2995 {
2996         struct g_mirror_disk *disk;
2997         struct g_provider *pp;
2998
2999         g_topology_assert_not();
3000         if (sc == NULL)
3001                 return (ENXIO);
3002         sx_assert(&sc->sc_lock, SX_XLOCKED);
3003
3004         pp = sc->sc_provider;
3005         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0 ||
3006             SCHEDULER_STOPPED())) {
3007                 switch (how) {
3008                 case G_MIRROR_DESTROY_SOFT:
3009                         G_MIRROR_DEBUG(1,
3010                             "Device %s is still open (r%dw%de%d).", pp->name,
3011                             pp->acr, pp->acw, pp->ace);
3012                         return (EBUSY);
3013                 case G_MIRROR_DESTROY_DELAYED:
3014                         G_MIRROR_DEBUG(1,
3015                             "Device %s will be destroyed on last close.",
3016                             pp->name);
3017                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
3018                                 if (disk->d_state ==
3019                                     G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3020                                         g_mirror_sync_stop(disk, 1);
3021                                 }
3022                         }
3023                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
3024                         return (EBUSY);
3025                 case G_MIRROR_DESTROY_HARD:
3026                         G_MIRROR_DEBUG(1, "Device %s is still open, so it "
3027                             "can't be definitely removed.", pp->name);
3028                 }
3029         }
3030
3031         g_topology_lock();
3032         if (sc->sc_geom->softc == NULL) {
3033                 g_topology_unlock();
3034                 return (0);
3035         }
3036         sc->sc_geom->softc = NULL;
3037         sc->sc_sync.ds_geom->softc = NULL;
3038         g_topology_unlock();
3039
3040         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3041         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
3042         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3043         sx_xunlock(&sc->sc_lock);
3044         mtx_lock(&sc->sc_queue_mtx);
3045         wakeup(sc);
3046         mtx_unlock(&sc->sc_queue_mtx);
3047         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3048         while (sc->sc_worker != NULL)
3049                 tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3050         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3051         sx_xlock(&sc->sc_lock);
3052         g_mirror_destroy_device(sc);
3053         free(sc, M_MIRROR);
3054         return (0);
3055 }
3056
3057 static void
3058 g_mirror_taste_orphan(struct g_consumer *cp)
3059 {
3060
3061         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3062             cp->provider->name));
3063 }
3064
3065 static struct g_geom *
3066 g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3067 {
3068         struct g_mirror_metadata md;
3069         struct g_mirror_softc *sc;
3070         struct g_consumer *cp;
3071         struct g_geom *gp;
3072         int error;
3073
3074         g_topology_assert();
3075         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3076         G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3077
3078         gp = g_new_geomf(mp, "mirror:taste");
3079         /*
3080          * This orphan function should be never called.
3081          */
3082         gp->orphan = g_mirror_taste_orphan;
3083         cp = g_new_consumer(gp);
3084         g_attach(cp, pp);
3085         error = g_mirror_read_metadata(cp, &md);
3086         g_detach(cp);
3087         g_destroy_consumer(cp);
3088         g_destroy_geom(gp);
3089         if (error != 0)
3090                 return (NULL);
3091         gp = NULL;
3092
3093         if (md.md_provider[0] != '\0' &&
3094             !g_compare_names(md.md_provider, pp->name))
3095                 return (NULL);
3096         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3097                 return (NULL);
3098         if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3099                 G_MIRROR_DEBUG(0,
3100                     "Device %s: provider %s marked as inactive, skipping.",
3101                     md.md_name, pp->name);
3102                 return (NULL);
3103         }
3104         if (g_mirror_debug >= 2)
3105                 mirror_metadata_dump(&md);
3106
3107         /*
3108          * Let's check if device already exists.
3109          */
3110         sc = NULL;
3111         LIST_FOREACH(gp, &mp->geom, geom) {
3112                 sc = gp->softc;
3113                 if (sc == NULL)
3114                         continue;
3115                 if (sc->sc_sync.ds_geom == gp)
3116                         continue;
3117                 if (strcmp(md.md_name, sc->sc_name) != 0)
3118                         continue;
3119                 if (md.md_mid != sc->sc_id) {
3120                         G_MIRROR_DEBUG(0, "Device %s already configured.",
3121                             sc->sc_name);
3122                         return (NULL);
3123                 }
3124                 break;
3125         }
3126         if (gp == NULL) {
3127                 gp = g_mirror_create(mp, &md);
3128                 if (gp == NULL) {
3129                         G_MIRROR_DEBUG(0, "Cannot create device %s.",
3130                             md.md_name);
3131                         return (NULL);
3132                 }
3133                 sc = gp->softc;
3134         }
3135         G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3136         g_topology_unlock();
3137         sx_xlock(&sc->sc_lock);
3138         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3139         error = g_mirror_add_disk(sc, pp, &md);
3140         if (error != 0) {
3141                 G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3142                     pp->name, gp->name, error);
3143                 if (LIST_EMPTY(&sc->sc_disks)) {
3144                         g_cancel_event(sc);
3145                         g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3146                         g_topology_lock();
3147                         return (NULL);
3148                 }
3149                 gp = NULL;
3150         }
3151         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3152         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3153                 g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3154                 g_topology_lock();
3155                 return (NULL);
3156         }
3157         sx_xunlock(&sc->sc_lock);
3158         g_topology_lock();
3159         return (gp);
3160 }
3161
3162 static void
3163 g_mirror_resize(struct g_consumer *cp)
3164 {
3165         struct g_mirror_disk *disk;
3166
3167         g_topology_assert();
3168         g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3169
3170         disk = cp->private;
3171         if (disk == NULL)
3172                 return;
3173         g_topology_unlock();
3174         g_mirror_update_metadata(disk);
3175         g_topology_lock();
3176 }
3177
3178 static int
3179 g_mirror_destroy_geom(struct gctl_req *req __unused,
3180     struct g_class *mp __unused, struct g_geom *gp)
3181 {
3182         struct g_mirror_softc *sc;
3183         int error;
3184
3185         g_topology_unlock();
3186         sc = gp->softc;
3187         sx_xlock(&sc->sc_lock);
3188         g_cancel_event(sc);
3189         error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3190         if (error != 0)
3191                 sx_xunlock(&sc->sc_lock);
3192         g_topology_lock();
3193         return (error);
3194 }
3195
3196 static void
3197 g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3198     struct g_consumer *cp, struct g_provider *pp)
3199 {
3200         struct g_mirror_softc *sc;
3201
3202         g_topology_assert();
3203
3204         sc = gp->softc;
3205         if (sc == NULL)
3206                 return;
3207         /* Skip synchronization geom. */
3208         if (gp == sc->sc_sync.ds_geom)
3209                 return;
3210         if (pp != NULL) {
3211                 /* Nothing here. */
3212         } else if (cp != NULL) {
3213                 struct g_mirror_disk *disk;
3214
3215                 disk = cp->private;
3216                 if (disk == NULL)
3217                         return;
3218                 g_topology_unlock();
3219                 sx_xlock(&sc->sc_lock);
3220                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3221                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3222                         sbuf_printf(sb, "%s<Synchronized>", indent);
3223                         if (disk->d_sync.ds_offset == 0)
3224                                 sbuf_printf(sb, "0%%");
3225                         else {
3226                                 sbuf_printf(sb, "%u%%",
3227                                     (u_int)((disk->d_sync.ds_offset * 100) /
3228                                     sc->sc_provider->mediasize));
3229                         }
3230                         sbuf_printf(sb, "</Synchronized>\n");
3231                         if (disk->d_sync.ds_offset > 0) {
3232                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3233                                     "</BytesSynced>\n", indent,
3234                                     (intmax_t)disk->d_sync.ds_offset);
3235                         }
3236                 }
3237                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3238                     disk->d_sync.ds_syncid);
3239                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3240                     disk->d_genid);
3241                 sbuf_printf(sb, "%s<Flags>", indent);
3242                 if (disk->d_flags == 0)
3243                         sbuf_printf(sb, "NONE");
3244                 else {
3245                         int first = 1;
3246
3247 #define ADD_FLAG(flag, name)    do {                                    \
3248         if ((disk->d_flags & (flag)) != 0) {                            \
3249                 if (!first)                                             \
3250                         sbuf_printf(sb, ", ");                          \
3251                 else                                                    \
3252                         first = 0;                                      \
3253                 sbuf_printf(sb, name);                                  \
3254         }                                                               \
3255 } while (0)
3256                         ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3257                         ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3258                         ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3259                         ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3260                             "SYNCHRONIZING");
3261                         ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3262                         ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3263 #undef  ADD_FLAG
3264                 }
3265                 sbuf_printf(sb, "</Flags>\n");
3266                 sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3267                     disk->d_priority);
3268                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3269                     g_mirror_disk_state2str(disk->d_state));
3270                 sx_xunlock(&sc->sc_lock);
3271                 g_topology_lock();
3272         } else {
3273                 g_topology_unlock();
3274                 sx_xlock(&sc->sc_lock);
3275                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3276                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3277                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3278                 sbuf_printf(sb, "%s<Flags>", indent);
3279                 if (sc->sc_flags == 0)
3280                         sbuf_printf(sb, "NONE");
3281                 else {
3282                         int first = 1;
3283
3284 #define ADD_FLAG(flag, name)    do {                                    \
3285         if ((sc->sc_flags & (flag)) != 0) {                             \
3286                 if (!first)                                             \
3287                         sbuf_printf(sb, ", ");                          \
3288                 else                                                    \
3289                         first = 0;                                      \
3290                 sbuf_printf(sb, name);                                  \
3291         }                                                               \
3292 } while (0)
3293                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3294                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3295 #undef  ADD_FLAG
3296                 }
3297                 sbuf_printf(sb, "</Flags>\n");
3298                 sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3299                     (u_int)sc->sc_slice);
3300                 sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3301                     balance_name(sc->sc_balance));
3302                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3303                     sc->sc_ndisks);
3304                 sbuf_printf(sb, "%s<State>", indent);
3305                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3306                         sbuf_printf(sb, "%s", "STARTING");
3307                 else if (sc->sc_ndisks ==
3308                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3309                         sbuf_printf(sb, "%s", "COMPLETE");
3310                 else
3311                         sbuf_printf(sb, "%s", "DEGRADED");
3312                 sbuf_printf(sb, "</State>\n");
3313                 sx_xunlock(&sc->sc_lock);
3314                 g_topology_lock();
3315         }
3316 }
3317
3318 static void
3319 g_mirror_shutdown_post_sync(void *arg, int howto)
3320 {
3321         struct g_class *mp;
3322         struct g_geom *gp, *gp2;
3323         struct g_mirror_softc *sc;
3324         int error;
3325
3326         mp = arg;
3327         g_topology_lock();
3328         g_mirror_shutdown = 1;
3329         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3330                 if ((sc = gp->softc) == NULL)
3331                         continue;
3332                 /* Skip synchronization geom. */
3333                 if (gp == sc->sc_sync.ds_geom)
3334                         continue;
3335                 g_topology_unlock();
3336                 sx_xlock(&sc->sc_lock);
3337                 g_mirror_idle(sc, -1);
3338                 g_cancel_event(sc);
3339                 error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3340                 if (error != 0)
3341                         sx_xunlock(&sc->sc_lock);
3342                 g_topology_lock();
3343         }
3344         g_topology_unlock();
3345 }
3346
3347 static void
3348 g_mirror_init(struct g_class *mp)
3349 {
3350
3351         g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3352             g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3353         if (g_mirror_post_sync == NULL)
3354                 G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3355 }
3356
3357 static void
3358 g_mirror_fini(struct g_class *mp)
3359 {
3360
3361         if (g_mirror_post_sync != NULL)
3362                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3363 }
3364
3365 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);