sys/geom/mirror/g_mirror.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/bio.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/fail.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/limits.h>
  40 #include <sys/lock.h>
  41 #include <sys/malloc.h>
  42 #include <sys/mutex.h>
  43 #include <sys/proc.h>
  44 #include <sys/sbuf.h>
  45 #include <sys/sched.h>
  46 #include <sys/sx.h>
  47 #include <sys/sysctl.h>
  48
  49 #include <geom/geom.h>
  50 #include <geom/geom_dbg.h>
  51 #include <geom/mirror/g_mirror.h>
  52
  53 FEATURE(geom_mirror, "GEOM mirroring support");
  54
  55 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
  56
  57 SYSCTL_DECL(_kern_geom);
  58 static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  59     "GEOM_MIRROR stuff");
  60 int g_mirror_debug = 0;
  61 SYSCTL_INT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RWTUN, &g_mirror_debug, 0,
  62     "Debug level");
  63 bool g_launch_mirror_before_timeout = true;
  64 SYSCTL_BOOL(_kern_geom_mirror, OID_AUTO, launch_mirror_before_timeout,
  65     CTLFLAG_RWTUN, &g_launch_mirror_before_timeout, 0,
  66     "If false, force gmirror to wait out the full kern.geom.mirror.timeout "
  67     "before launching mirrors");
  68 static u_int g_mirror_timeout = 4;
  69 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RWTUN, &g_mirror_timeout,
  70     0, "Time to wait on all mirror components");
  71 static u_int g_mirror_idletime = 5;
  72 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RWTUN,
  73     &g_mirror_idletime, 0, "Mark components as clean when idling");
  74 static u_int g_mirror_disconnect_on_failure = 1;
  75 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RWTUN,
  76     &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  77 static u_int g_mirror_syncreqs = 2;
  78 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  79     &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
  80 static u_int g_mirror_sync_period = 5;
  81 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_update_period, CTLFLAG_RWTUN,
  82     &g_mirror_sync_period, 0,
  83     "Metadata update period during synchronization, in seconds");
  84
  85 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  86         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));       \
  87         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  88         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));       \
  89 } while (0)
  90
  91 static eventhandler_tag g_mirror_post_sync = NULL;
  92 static int g_mirror_shutdown = 0;
  93
  94 static g_ctl_destroy_geom_t g_mirror_destroy_geom;
  95 static g_taste_t g_mirror_taste;
  96 static g_init_t g_mirror_init;
  97 static g_fini_t g_mirror_fini;
  98 static g_provgone_t g_mirror_providergone;
  99 static g_resize_t g_mirror_resize;
 100
 101 struct g_class g_mirror_class = {
 102         .name = G_MIRROR_CLASS_NAME,
 103         .version = G_VERSION,
 104         .ctlreq = g_mirror_config,
 105         .taste = g_mirror_taste,
 106         .destroy_geom = g_mirror_destroy_geom,
 107         .init = g_mirror_init,
 108         .fini = g_mirror_fini,
 109         .providergone = g_mirror_providergone,
 110         .resize = g_mirror_resize
 111 };
 112
 113 static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
 114 static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
 115 static void g_mirror_update_device(struct g_mirror_softc *sc, bool force);
 116 static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
 117     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 118 static int g_mirror_refresh_device(struct g_mirror_softc *sc,
 119     const struct g_provider *pp, const struct g_mirror_metadata *md);
 120 static void g_mirror_sync_reinit(const struct g_mirror_disk *disk,
 121     struct bio *bp, off_t offset);
 122 static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
 123 static void g_mirror_register_request(struct g_mirror_softc *sc,
 124     struct bio *bp);
 125 static void g_mirror_sync_release(struct g_mirror_softc *sc);
 126
 127 static const char *
 128 g_mirror_disk_state2str(int state)
 129 {
 130
 131         switch (state) {
 132         case G_MIRROR_DISK_STATE_NONE:
 133                 return ("NONE");
 134         case G_MIRROR_DISK_STATE_NEW:
 135                 return ("NEW");
 136         case G_MIRROR_DISK_STATE_ACTIVE:
 137                 return ("ACTIVE");
 138         case G_MIRROR_DISK_STATE_STALE:
 139                 return ("STALE");
 140         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 141                 return ("SYNCHRONIZING");
 142         case G_MIRROR_DISK_STATE_DISCONNECTED:
 143                 return ("DISCONNECTED");
 144         case G_MIRROR_DISK_STATE_DESTROY:
 145                 return ("DESTROY");
 146         default:
 147                 return ("INVALID");
 148         }
 149 }
 150
 151 static const char *
 152 g_mirror_device_state2str(int state)
 153 {
 154
 155         switch (state) {
 156         case G_MIRROR_DEVICE_STATE_STARTING:
 157                 return ("STARTING");
 158         case G_MIRROR_DEVICE_STATE_RUNNING:
 159                 return ("RUNNING");
 160         default:
 161                 return ("INVALID");
 162         }
 163 }
 164
 165 static const char *
 166 g_mirror_get_diskname(struct g_mirror_disk *disk)
 167 {
 168
 169         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 170                 return ("[unknown]");
 171         return (disk->d_name);
 172 }
 173
 174 /*
 175  * --- Events handling functions ---
 176  * Events in geom_mirror are used to maintain disks and device status
 177  * from one thread to simplify locking.
 178  */
 179 static void
 180 g_mirror_event_free(struct g_mirror_event *ep)
 181 {
 182
 183         free(ep, M_MIRROR);
 184 }
 185
 186 int
 187 g_mirror_event_send(void *arg, int state, int flags)
 188 {
 189         struct g_mirror_softc *sc;
 190         struct g_mirror_disk *disk;
 191         struct g_mirror_event *ep;
 192         int error;
 193
 194         ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
 195         G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 196         if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
 197                 disk = NULL;
 198                 sc = arg;
 199         } else {
 200                 disk = arg;
 201                 sc = disk->d_softc;
 202         }
 203         ep->e_disk = disk;
 204         ep->e_state = state;
 205         ep->e_flags = flags;
 206         ep->e_error = 0;
 207         mtx_lock(&sc->sc_events_mtx);
 208         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 209         mtx_unlock(&sc->sc_events_mtx);
 210         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 211         mtx_lock(&sc->sc_queue_mtx);
 212         wakeup(sc);
 213         mtx_unlock(&sc->sc_queue_mtx);
 214         if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 215                 return (0);
 216         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 217         sx_xunlock(&sc->sc_lock);
 218         while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
 219                 mtx_lock(&sc->sc_events_mtx);
 220                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
 221                     hz * 5);
 222         }
 223         error = ep->e_error;
 224         g_mirror_event_free(ep);
 225         sx_xlock(&sc->sc_lock);
 226         return (error);
 227 }
 228
 229 static struct g_mirror_event *
 230 g_mirror_event_first(struct g_mirror_softc *sc)
 231 {
 232         struct g_mirror_event *ep;
 233
 234         mtx_lock(&sc->sc_events_mtx);
 235         ep = TAILQ_FIRST(&sc->sc_events);
 236         mtx_unlock(&sc->sc_events_mtx);
 237         return (ep);
 238 }
 239
 240 static void
 241 g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
 242 {
 243
 244         mtx_lock(&sc->sc_events_mtx);
 245         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 246         mtx_unlock(&sc->sc_events_mtx);
 247 }
 248
 249 static void
 250 g_mirror_event_cancel(struct g_mirror_disk *disk)
 251 {
 252         struct g_mirror_softc *sc;
 253         struct g_mirror_event *ep, *tmpep;
 254
 255         sc = disk->d_softc;
 256         sx_assert(&sc->sc_lock, SX_XLOCKED);
 257
 258         mtx_lock(&sc->sc_events_mtx);
 259         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 260                 if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
 261                         continue;
 262                 if (ep->e_disk != disk)
 263                         continue;
 264                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 265                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 266                         g_mirror_event_free(ep);
 267                 else {
 268                         ep->e_error = ECANCELED;
 269                         wakeup(ep);
 270                 }
 271         }
 272         mtx_unlock(&sc->sc_events_mtx);
 273 }
 274
 275 /*
 276  * Return the number of disks in given state.
 277  * If state is equal to -1, count all connected disks.
 278  */
 279 u_int
 280 g_mirror_ndisks(struct g_mirror_softc *sc, int state)
 281 {
 282         struct g_mirror_disk *disk;
 283         u_int n = 0;
 284
 285         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 286                 if (state == -1 || disk->d_state == state)
 287                         n++;
 288         }
 289         return (n);
 290 }
 291
 292 /*
 293  * Find a disk in mirror by its disk ID.
 294  */
 295 static struct g_mirror_disk *
 296 g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
 297 {
 298         struct g_mirror_disk *disk;
 299
 300         sx_assert(&sc->sc_lock, SX_XLOCKED);
 301
 302         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 303                 if (disk->d_id == id)
 304                         return (disk);
 305         }
 306         return (NULL);
 307 }
 308
 309 static u_int
 310 g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
 311 {
 312         struct bio *bp;
 313         u_int nreqs = 0;
 314
 315         mtx_lock(&sc->sc_queue_mtx);
 316         TAILQ_FOREACH(bp, &sc->sc_queue, bio_queue) {
 317                 if (bp->bio_from == cp)
 318                         nreqs++;
 319         }
 320         mtx_unlock(&sc->sc_queue_mtx);
 321         return (nreqs);
 322 }
 323
 324 static int
 325 g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
 326 {
 327
 328         if (cp->index > 0) {
 329                 G_MIRROR_DEBUG(2,
 330                     "I/O requests for %s exist, can't destroy it now.",
 331                     cp->provider->name);
 332                 return (1);
 333         }
 334         if (g_mirror_nrequests(sc, cp) > 0) {
 335                 G_MIRROR_DEBUG(2,
 336                     "I/O requests for %s in queue, can't destroy it now.",
 337                     cp->provider->name);
 338                 return (1);
 339         }
 340         return (0);
 341 }
 342
 343 static void
 344 g_mirror_destroy_consumer(void *arg, int flags __unused)
 345 {
 346         struct g_consumer *cp;
 347
 348         g_topology_assert();
 349
 350         cp = arg;
 351         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 352         g_detach(cp);
 353         g_destroy_consumer(cp);
 354 }
 355
 356 static void
 357 g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 358 {
 359         struct g_provider *pp;
 360         int retaste_wait;
 361
 362         g_topology_assert();
 363
 364         cp->private = NULL;
 365         if (g_mirror_is_busy(sc, cp))
 366                 return;
 367         pp = cp->provider;
 368         retaste_wait = 0;
 369         if (cp->acw == 1) {
 370                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 371                         retaste_wait = 1;
 372         }
 373         G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 374             -cp->acw, -cp->ace, 0);
 375         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 376                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 377         if (retaste_wait) {
 378                 /*
 379                  * After retaste event was send (inside g_access()), we can send
 380                  * event to detach and destroy consumer.
 381                  * A class, which has consumer to the given provider connected
 382                  * will not receive retaste event for the provider.
 383                  * This is the way how I ignore retaste events when I close
 384                  * consumers opened for write: I detach and destroy consumer
 385                  * after retaste event is sent.
 386                  */
 387                 g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
 388                 return;
 389         }
 390         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
 391         g_detach(cp);
 392         g_destroy_consumer(cp);
 393 }
 394
 395 static int
 396 g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
 397 {
 398         struct g_consumer *cp;
 399         int error;
 400
 401         g_topology_assert_not();
 402         KASSERT(disk->d_consumer == NULL,
 403             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 404
 405         g_topology_lock();
 406         cp = g_new_consumer(disk->d_softc->sc_geom);
 407         cp->flags |= G_CF_DIRECT_RECEIVE;
 408         error = g_attach(cp, pp);
 409         if (error != 0) {
 410                 g_destroy_consumer(cp);
 411                 g_topology_unlock();
 412                 return (error);
 413         }
 414         error = g_access(cp, 1, 1, 1);
 415         if (error != 0) {
 416                 g_detach(cp);
 417                 g_destroy_consumer(cp);
 418                 g_topology_unlock();
 419                 G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
 420                     pp->name, error);
 421                 return (error);
 422         }
 423         g_topology_unlock();
 424         disk->d_consumer = cp;
 425         disk->d_consumer->private = disk;
 426         disk->d_consumer->index = 0;
 427
 428         G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
 429         return (0);
 430 }
 431
 432 static void
 433 g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 434 {
 435
 436         g_topology_assert();
 437
 438         if (cp == NULL)
 439                 return;
 440         if (cp->provider != NULL)
 441                 g_mirror_kill_consumer(sc, cp);
 442         else
 443                 g_destroy_consumer(cp);
 444 }
 445
 446 /*
 447  * Initialize disk. This means allocate memory, create consumer, attach it
 448  * to the provider and open access (r1w1e1) to it.
 449  */
 450 static struct g_mirror_disk *
 451 g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 452     struct g_mirror_metadata *md, int *errorp)
 453 {
 454         struct g_mirror_disk *disk;
 455         int i, error;
 456
 457         disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
 458         if (disk == NULL) {
 459                 error = ENOMEM;
 460                 goto fail;
 461         }
 462         disk->d_softc = sc;
 463         error = g_mirror_connect_disk(disk, pp);
 464         if (error != 0)
 465                 goto fail;
 466         disk->d_id = md->md_did;
 467         disk->d_state = G_MIRROR_DISK_STATE_NONE;
 468         disk->d_priority = md->md_priority;
 469         disk->d_flags = md->md_dflags;
 470         error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
 471         if (error == 0 && i != 0)
 472                 disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 473         if (md->md_provider[0] != '\0')
 474                 disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
 475         disk->d_sync.ds_consumer = NULL;
 476         disk->d_sync.ds_offset = md->md_sync_offset;
 477         disk->d_sync.ds_offset_done = md->md_sync_offset;
 478         disk->d_sync.ds_update_ts = time_uptime;
 479         disk->d_genid = md->md_genid;
 480         disk->d_sync.ds_syncid = md->md_syncid;
 481         disk->d_init_ndisks = md->md_all;
 482         disk->d_init_slice = md->md_slice;
 483         disk->d_init_balance = md->md_balance;
 484         disk->d_init_mediasize = md->md_mediasize;
 485         if (errorp != NULL)
 486                 *errorp = 0;
 487         return (disk);
 488 fail:
 489         if (errorp != NULL)
 490                 *errorp = error;
 491         if (disk != NULL)
 492                 free(disk, M_MIRROR);
 493         return (NULL);
 494 }
 495
 496 static void
 497 g_mirror_destroy_disk(struct g_mirror_disk *disk)
 498 {
 499         struct g_mirror_softc *sc;
 500
 501         g_topology_assert_not();
 502         sc = disk->d_softc;
 503         sx_assert(&sc->sc_lock, SX_XLOCKED);
 504
 505         g_topology_lock();
 506         LIST_REMOVE(disk, d_next);
 507         g_topology_unlock();
 508         g_mirror_event_cancel(disk);
 509         if (sc->sc_hint == disk)
 510                 sc->sc_hint = NULL;
 511         switch (disk->d_state) {
 512         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 513                 g_mirror_sync_stop(disk, 1);
 514                 /* FALLTHROUGH */
 515         case G_MIRROR_DISK_STATE_NEW:
 516         case G_MIRROR_DISK_STATE_STALE:
 517         case G_MIRROR_DISK_STATE_ACTIVE:
 518                 g_topology_lock();
 519                 g_mirror_disconnect_consumer(sc, disk->d_consumer);
 520                 g_topology_unlock();
 521                 free(disk, M_MIRROR);
 522                 break;
 523         default:
 524                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 525                     g_mirror_get_diskname(disk),
 526                     g_mirror_disk_state2str(disk->d_state)));
 527         }
 528 }
 529
 530 static void
 531 g_mirror_free_device(struct g_mirror_softc *sc)
 532 {
 533
 534         g_topology_assert();
 535
 536         mtx_destroy(&sc->sc_queue_mtx);
 537         mtx_destroy(&sc->sc_events_mtx);
 538         mtx_destroy(&sc->sc_done_mtx);
 539         sx_destroy(&sc->sc_lock);
 540         free(sc, M_MIRROR);
 541 }
 542
 543 static void
 544 g_mirror_providergone(struct g_provider *pp)
 545 {
 546         struct g_mirror_softc *sc = pp->private;
 547
 548         if ((--sc->sc_refcnt) == 0)
 549                 g_mirror_free_device(sc);
 550 }
 551
 552 static void
 553 g_mirror_destroy_device(struct g_mirror_softc *sc)
 554 {
 555         struct g_mirror_disk *disk;
 556         struct g_mirror_event *ep;
 557         struct g_geom *gp;
 558         struct g_consumer *cp, *tmpcp;
 559
 560         g_topology_assert_not();
 561         sx_assert(&sc->sc_lock, SX_XLOCKED);
 562
 563         gp = sc->sc_geom;
 564         if (sc->sc_provider != NULL)
 565                 g_mirror_destroy_provider(sc);
 566         for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
 567             disk = LIST_FIRST(&sc->sc_disks)) {
 568                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 569                 g_mirror_update_metadata(disk);
 570                 g_mirror_destroy_disk(disk);
 571         }
 572         while ((ep = g_mirror_event_first(sc)) != NULL) {
 573                 g_mirror_event_remove(sc, ep);
 574                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 575                         g_mirror_event_free(ep);
 576                 else {
 577                         ep->e_error = ECANCELED;
 578                         ep->e_flags |= G_MIRROR_EVENT_DONE;
 579                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 580                         mtx_lock(&sc->sc_events_mtx);
 581                         wakeup(ep);
 582                         mtx_unlock(&sc->sc_events_mtx);
 583                 }
 584         }
 585         callout_drain(&sc->sc_callout);
 586
 587         g_topology_lock();
 588         LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
 589                 g_mirror_disconnect_consumer(sc, cp);
 590         }
 591         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 592         G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
 593         g_wither_geom(gp, ENXIO);
 594         sx_xunlock(&sc->sc_lock);
 595         if ((--sc->sc_refcnt) == 0)
 596                 g_mirror_free_device(sc);
 597         g_topology_unlock();
 598 }
 599
 600 static void
 601 g_mirror_orphan(struct g_consumer *cp)
 602 {
 603         struct g_mirror_disk *disk;
 604
 605         g_topology_assert();
 606
 607         disk = cp->private;
 608         if (disk == NULL)
 609                 return;
 610         disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 611         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 612             G_MIRROR_EVENT_DONTWAIT);
 613 }
 614
 615 /*
 616  * Function should return the next active disk on the list.
 617  * It is possible that it will be the same disk as given.
 618  * If there are no active disks on list, NULL is returned.
 619  */
 620 static __inline struct g_mirror_disk *
 621 g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
 622 {
 623         struct g_mirror_disk *dp;
 624
 625         for (dp = LIST_NEXT(disk, d_next); dp != disk;
 626             dp = LIST_NEXT(dp, d_next)) {
 627                 if (dp == NULL)
 628                         dp = LIST_FIRST(&sc->sc_disks);
 629                 if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
 630                         break;
 631         }
 632         if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 633                 return (NULL);
 634         return (dp);
 635 }
 636
 637 static struct g_mirror_disk *
 638 g_mirror_get_disk(struct g_mirror_softc *sc)
 639 {
 640         struct g_mirror_disk *disk;
 641
 642         if (sc->sc_hint == NULL) {
 643                 sc->sc_hint = LIST_FIRST(&sc->sc_disks);
 644                 if (sc->sc_hint == NULL)
 645                         return (NULL);
 646         }
 647         disk = sc->sc_hint;
 648         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
 649                 disk = g_mirror_find_next(sc, disk);
 650                 if (disk == NULL)
 651                         return (NULL);
 652         }
 653         sc->sc_hint = g_mirror_find_next(sc, disk);
 654         return (disk);
 655 }
 656
 657 static int
 658 g_mirror_write_metadata(struct g_mirror_disk *disk,
 659     struct g_mirror_metadata *md)
 660 {
 661         struct g_mirror_softc *sc;
 662         struct g_consumer *cp;
 663         off_t offset, length;
 664         u_char *sector;
 665         int error = 0;
 666
 667         g_topology_assert_not();
 668         sc = disk->d_softc;
 669         sx_assert(&sc->sc_lock, SX_LOCKED);
 670
 671         cp = disk->d_consumer;
 672         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 673         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 674         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 675             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 676             cp->acw, cp->ace));
 677         length = cp->provider->sectorsize;
 678         offset = cp->provider->mediasize - length;
 679         sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
 680         if (md != NULL &&
 681             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
 682                 /*
 683                  * Handle the case, when the size of parent provider reduced.
 684                  */
 685                 if (offset < md->md_mediasize)
 686                         error = ENOSPC;
 687                 else
 688                         mirror_metadata_encode(md, sector);
 689         }
 690         KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_metadata_write, error);
 691         if (error == 0)
 692                 error = g_write_data(cp, offset, sector, length);
 693         free(sector, M_MIRROR);
 694         if (error != 0) {
 695                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 696                         disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 697                         G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
 698                             "(device=%s, error=%d).",
 699                             g_mirror_get_diskname(disk), sc->sc_name, error);
 700                 } else {
 701                         G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
 702                             "(device=%s, error=%d).",
 703                             g_mirror_get_diskname(disk), sc->sc_name, error);
 704                 }
 705                 if (g_mirror_disconnect_on_failure &&
 706                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 707                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 708                         g_mirror_event_send(disk,
 709                             G_MIRROR_DISK_STATE_DISCONNECTED,
 710                             G_MIRROR_EVENT_DONTWAIT);
 711                 }
 712         }
 713         return (error);
 714 }
 715
 716 static int
 717 g_mirror_clear_metadata(struct g_mirror_disk *disk)
 718 {
 719         int error;
 720
 721         g_topology_assert_not();
 722         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 723
 724         if (disk->d_softc->sc_type != G_MIRROR_TYPE_AUTOMATIC)
 725                 return (0);
 726         error = g_mirror_write_metadata(disk, NULL);
 727         if (error == 0) {
 728                 G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
 729                     g_mirror_get_diskname(disk));
 730         } else {
 731                 G_MIRROR_DEBUG(0,
 732                     "Cannot clear metadata on disk %s (error=%d).",
 733                     g_mirror_get_diskname(disk), error);
 734         }
 735         return (error);
 736 }
 737
 738 void
 739 g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
 740     struct g_mirror_metadata *md)
 741 {
 742
 743         strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
 744         md->md_version = G_MIRROR_VERSION;
 745         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 746         md->md_mid = sc->sc_id;
 747         md->md_all = sc->sc_ndisks;
 748         md->md_slice = sc->sc_slice;
 749         md->md_balance = sc->sc_balance;
 750         md->md_genid = sc->sc_genid;
 751         md->md_mediasize = sc->sc_mediasize;
 752         md->md_sectorsize = sc->sc_sectorsize;
 753         md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
 754         bzero(md->md_provider, sizeof(md->md_provider));
 755         if (disk == NULL) {
 756                 md->md_did = arc4random();
 757                 md->md_priority = 0;
 758                 md->md_syncid = 0;
 759                 md->md_dflags = 0;
 760                 md->md_sync_offset = 0;
 761                 md->md_provsize = 0;
 762         } else {
 763                 md->md_did = disk->d_id;
 764                 md->md_priority = disk->d_priority;
 765                 md->md_syncid = disk->d_sync.ds_syncid;
 766                 md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
 767                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
 768                         md->md_sync_offset = disk->d_sync.ds_offset_done;
 769                 else
 770                         md->md_sync_offset = 0;
 771                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
 772                         strlcpy(md->md_provider,
 773                             disk->d_consumer->provider->name,
 774                             sizeof(md->md_provider));
 775                 }
 776                 md->md_provsize = disk->d_consumer->provider->mediasize;
 777         }
 778 }
 779
 780 void
 781 g_mirror_update_metadata(struct g_mirror_disk *disk)
 782 {
 783         struct g_mirror_softc *sc;
 784         struct g_mirror_metadata md;
 785         int error;
 786
 787         g_topology_assert_not();
 788         sc = disk->d_softc;
 789         sx_assert(&sc->sc_lock, SX_LOCKED);
 790
 791         if (sc->sc_type != G_MIRROR_TYPE_AUTOMATIC)
 792                 return;
 793         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
 794                 g_mirror_fill_metadata(sc, disk, &md);
 795         error = g_mirror_write_metadata(disk, &md);
 796         if (error == 0) {
 797                 G_MIRROR_DEBUG(2, "Metadata on %s updated.",
 798                     g_mirror_get_diskname(disk));
 799         } else {
 800                 G_MIRROR_DEBUG(0,
 801                     "Cannot update metadata on disk %s (error=%d).",
 802                     g_mirror_get_diskname(disk), error);
 803         }
 804 }
 805
 806 static void
 807 g_mirror_bump_syncid(struct g_mirror_softc *sc)
 808 {
 809         struct g_mirror_disk *disk;
 810
 811         g_topology_assert_not();
 812         sx_assert(&sc->sc_lock, SX_XLOCKED);
 813         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 814             ("%s called with no active disks (device=%s).", __func__,
 815             sc->sc_name));
 816
 817         sc->sc_syncid++;
 818         G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 819             sc->sc_syncid);
 820         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 821                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 822                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 823                         disk->d_sync.ds_syncid = sc->sc_syncid;
 824                         g_mirror_update_metadata(disk);
 825                 }
 826         }
 827 }
 828
 829 static void
 830 g_mirror_bump_genid(struct g_mirror_softc *sc)
 831 {
 832         struct g_mirror_disk *disk;
 833
 834         g_topology_assert_not();
 835         sx_assert(&sc->sc_lock, SX_XLOCKED);
 836         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 837             ("%s called with no active disks (device=%s).", __func__,
 838             sc->sc_name));
 839
 840         sc->sc_genid++;
 841         G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 842             sc->sc_genid);
 843         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 844                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 845                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 846                         disk->d_genid = sc->sc_genid;
 847                         g_mirror_update_metadata(disk);
 848                 }
 849         }
 850 }
 851
 852 static int
 853 g_mirror_idle(struct g_mirror_softc *sc, int acw)
 854 {
 855         struct g_mirror_disk *disk;
 856         int timeout;
 857
 858         g_topology_assert_not();
 859         sx_assert(&sc->sc_lock, SX_XLOCKED);
 860
 861         if (sc->sc_provider == NULL)
 862                 return (0);
 863         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 864                 return (0);
 865         if (sc->sc_idle)
 866                 return (0);
 867         if (sc->sc_writes > 0)
 868                 return (0);
 869         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 870                 timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
 871                 if (!g_mirror_shutdown && timeout > 0)
 872                         return (timeout);
 873         }
 874         sc->sc_idle = 1;
 875         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 876                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 877                         continue;
 878                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as clean.",
 879                     g_mirror_get_diskname(disk), sc->sc_name);
 880                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 881                 g_mirror_update_metadata(disk);
 882         }
 883         return (0);
 884 }
 885
 886 static void
 887 g_mirror_unidle(struct g_mirror_softc *sc)
 888 {
 889         struct g_mirror_disk *disk;
 890
 891         g_topology_assert_not();
 892         sx_assert(&sc->sc_lock, SX_XLOCKED);
 893
 894         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 895                 return;
 896         sc->sc_idle = 0;
 897         sc->sc_last_write = time_uptime;
 898         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 899                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 900                         continue;
 901                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as dirty.",
 902                     g_mirror_get_diskname(disk), sc->sc_name);
 903                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
 904                 g_mirror_update_metadata(disk);
 905         }
 906 }
 907
 908 static void
 909 g_mirror_done(struct bio *bp)
 910 {
 911         struct g_mirror_softc *sc;
 912
 913         sc = bp->bio_from->geom->softc;
 914         bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
 915         mtx_lock(&sc->sc_queue_mtx);
 916         TAILQ_INSERT_TAIL(&sc->sc_queue, bp, bio_queue);
 917         mtx_unlock(&sc->sc_queue_mtx);
 918         wakeup(sc);
 919 }
 920
 921 static void
 922 g_mirror_regular_request_error(struct g_mirror_softc *sc,
 923     struct g_mirror_disk *disk, struct bio *bp)
 924 {
 925
 926         if ((bp->bio_cmd == BIO_FLUSH || bp->bio_cmd == BIO_SPEEDUP) &&
 927             bp->bio_error == EOPNOTSUPP)
 928                 return;
 929
 930         if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 931                 disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 932                 G_MIRROR_LOGREQ(0, bp, "Request failed (error=%d).",
 933                     bp->bio_error);
 934         } else {
 935                 G_MIRROR_LOGREQ(1, bp, "Request failed (error=%d).",
 936                     bp->bio_error);
 937         }
 938         if (g_mirror_disconnect_on_failure &&
 939             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 940                 if (bp->bio_error == ENXIO &&
 941                     bp->bio_cmd == BIO_READ)
 942                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 943                 else if (bp->bio_error == ENXIO)
 944                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID_NOW;
 945                 else
 946                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 947                 g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 948                     G_MIRROR_EVENT_DONTWAIT);
 949         }
 950 }
 951
 952 static void
 953 g_mirror_regular_request(struct g_mirror_softc *sc, struct bio *bp)
 954 {
 955         struct g_mirror_disk *disk;
 956         struct bio *pbp;
 957
 958         g_topology_assert_not();
 959         KASSERT(sc->sc_provider == bp->bio_parent->bio_to,
 960             ("regular request %p with unexpected origin", bp));
 961
 962         pbp = bp->bio_parent;
 963         bp->bio_from->index--;
 964         if (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_DELETE)
 965                 sc->sc_writes--;
 966         disk = bp->bio_from->private;
 967         if (disk == NULL) {
 968                 g_topology_lock();
 969                 g_mirror_kill_consumer(sc, bp->bio_from);
 970                 g_topology_unlock();
 971         }
 972
 973         switch (bp->bio_cmd) {
 974         case BIO_READ:
 975                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_regular_request_read,
 976                     bp->bio_error);
 977                 break;
 978         case BIO_WRITE:
 979                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_regular_request_write,
 980                     bp->bio_error);
 981                 break;
 982         case BIO_DELETE:
 983                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_regular_request_delete,
 984                     bp->bio_error);
 985                 break;
 986         case BIO_FLUSH:
 987                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_regular_request_flush,
 988                     bp->bio_error);
 989                 break;
 990         case BIO_SPEEDUP:
 991                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_regular_request_speedup,
 992                     bp->bio_error);
 993                 break;
 994         }
 995
 996         pbp->bio_inbed++;
 997         KASSERT(pbp->bio_inbed <= pbp->bio_children,
 998             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
 999             pbp->bio_children));
1000         if (bp->bio_error == 0 && pbp->bio_error == 0) {
1001                 G_MIRROR_LOGREQ(3, bp, "Request delivered.");
1002                 g_destroy_bio(bp);
1003                 if (pbp->bio_children == pbp->bio_inbed) {
1004                         G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
1005                         pbp->bio_completed = pbp->bio_length;
1006                         if (pbp->bio_cmd == BIO_WRITE ||
1007                             pbp->bio_cmd == BIO_DELETE) {
1008                                 TAILQ_REMOVE(&sc->sc_inflight, pbp, bio_queue);
1009                                 /* Release delayed sync requests if possible. */
1010                                 g_mirror_sync_release(sc);
1011                         }
1012                         g_io_deliver(pbp, pbp->bio_error);
1013                 }
1014                 return;
1015         } else if (bp->bio_error != 0) {
1016                 if (pbp->bio_error == 0)
1017                         pbp->bio_error = bp->bio_error;
1018                 if (disk != NULL)
1019                         g_mirror_regular_request_error(sc, disk, bp);
1020                 switch (pbp->bio_cmd) {
1021                 case BIO_DELETE:
1022                 case BIO_WRITE:
1023                 case BIO_FLUSH:
1024                 case BIO_SPEEDUP:
1025                         pbp->bio_inbed--;
1026                         pbp->bio_children--;
1027                         break;
1028                 }
1029         }
1030         g_destroy_bio(bp);
1031
1032         switch (pbp->bio_cmd) {
1033         case BIO_READ:
1034                 if (pbp->bio_inbed < pbp->bio_children)
1035                         break;
1036                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
1037                         g_io_deliver(pbp, pbp->bio_error);
1038                 else {
1039                         pbp->bio_error = 0;
1040                         mtx_lock(&sc->sc_queue_mtx);
1041                         TAILQ_INSERT_TAIL(&sc->sc_queue, pbp, bio_queue);
1042                         mtx_unlock(&sc->sc_queue_mtx);
1043                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1044                         wakeup(sc);
1045                 }
1046                 break;
1047         case BIO_DELETE:
1048         case BIO_WRITE:
1049         case BIO_FLUSH:
1050         case BIO_SPEEDUP:
1051                 if (pbp->bio_children == 0) {
1052                         /*
1053                          * All requests failed.
1054                          */
1055                 } else if (pbp->bio_inbed < pbp->bio_children) {
1056                         /* Do nothing. */
1057                         break;
1058                 } else if (pbp->bio_children == pbp->bio_inbed) {
1059                         /* Some requests succeeded. */
1060                         pbp->bio_error = 0;
1061                         pbp->bio_completed = pbp->bio_length;
1062                 }
1063                 if (pbp->bio_cmd == BIO_WRITE || pbp->bio_cmd == BIO_DELETE) {
1064                         TAILQ_REMOVE(&sc->sc_inflight, pbp, bio_queue);
1065                         /* Release delayed sync requests if possible. */
1066                         g_mirror_sync_release(sc);
1067                 }
1068                 g_io_deliver(pbp, pbp->bio_error);
1069                 break;
1070         default:
1071                 KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1072                 break;
1073         }
1074 }
1075
1076 static void
1077 g_mirror_sync_done(struct bio *bp)
1078 {
1079         struct g_mirror_softc *sc;
1080
1081         G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1082         sc = bp->bio_from->geom->softc;
1083         bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1084         mtx_lock(&sc->sc_queue_mtx);
1085         TAILQ_INSERT_TAIL(&sc->sc_queue, bp, bio_queue);
1086         mtx_unlock(&sc->sc_queue_mtx);
1087         wakeup(sc);
1088 }
1089
1090 static void
1091 g_mirror_candelete(struct bio *bp)
1092 {
1093         struct g_mirror_softc *sc;
1094         struct g_mirror_disk *disk;
1095         int val;
1096
1097         sc = bp->bio_to->private;
1098         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1099                 if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
1100                         break;
1101         }
1102         val = disk != NULL;
1103         g_handleattr(bp, "GEOM::candelete", &val, sizeof(val));
1104 }
1105
1106 static void
1107 g_mirror_kernel_dump(struct bio *bp)
1108 {
1109         struct g_mirror_softc *sc;
1110         struct g_mirror_disk *disk;
1111         struct bio *cbp;
1112         struct g_kerneldump *gkd;
1113
1114         /*
1115          * We configure dumping to the first component, because this component
1116          * will be used for reading with 'prefer' balance algorithm.
1117          * If the component with the highest priority is currently disconnected
1118          * we will not be able to read the dump after the reboot if it will be
1119          * connected and synchronized later. Can we do something better?
1120          */
1121         sc = bp->bio_to->private;
1122         disk = LIST_FIRST(&sc->sc_disks);
1123
1124         gkd = (struct g_kerneldump *)bp->bio_data;
1125         if (gkd->length > bp->bio_to->mediasize)
1126                 gkd->length = bp->bio_to->mediasize;
1127         cbp = g_clone_bio(bp);
1128         if (cbp == NULL) {
1129                 g_io_deliver(bp, ENOMEM);
1130                 return;
1131         }
1132         cbp->bio_done = g_std_done;
1133         g_io_request(cbp, disk->d_consumer);
1134         G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1135             g_mirror_get_diskname(disk));
1136 }
1137
1138 static void
1139 g_mirror_start(struct bio *bp)
1140 {
1141         struct g_mirror_softc *sc;
1142
1143         sc = bp->bio_to->private;
1144         /*
1145          * If sc == NULL or there are no valid disks, provider's error
1146          * should be set and g_mirror_start() should not be called at all.
1147          */
1148         KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1149             ("Provider's error should be set (error=%d)(mirror=%s).",
1150             bp->bio_to->error, bp->bio_to->name));
1151         G_MIRROR_LOGREQ(3, bp, "Request received.");
1152
1153         switch (bp->bio_cmd) {
1154         case BIO_READ:
1155         case BIO_WRITE:
1156         case BIO_DELETE:
1157         case BIO_SPEEDUP:
1158         case BIO_FLUSH:
1159                 break;
1160         case BIO_GETATTR:
1161                 if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
1162                         g_mirror_candelete(bp);
1163                         return;
1164                 } else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1165                         g_mirror_kernel_dump(bp);
1166                         return;
1167                 }
1168                 /* FALLTHROUGH */
1169         default:
1170                 g_io_deliver(bp, EOPNOTSUPP);
1171                 return;
1172         }
1173         mtx_lock(&sc->sc_queue_mtx);
1174         if (bp->bio_to->error != 0) {
1175                 mtx_unlock(&sc->sc_queue_mtx);
1176                 g_io_deliver(bp, bp->bio_to->error);
1177                 return;
1178         }
1179         TAILQ_INSERT_TAIL(&sc->sc_queue, bp, bio_queue);
1180         mtx_unlock(&sc->sc_queue_mtx);
1181         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1182         wakeup(sc);
1183 }
1184
1185 /*
1186  * Return TRUE if the given request is colliding with a in-progress
1187  * synchronization request.
1188  */
1189 static bool
1190 g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1191 {
1192         struct g_mirror_disk *disk;
1193         struct bio *sbp;
1194         off_t rstart, rend, sstart, send;
1195         u_int i;
1196
1197         if (sc->sc_sync.ds_ndisks == 0)
1198                 return (false);
1199         rstart = bp->bio_offset;
1200         rend = bp->bio_offset + bp->bio_length;
1201         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1202                 if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1203                         continue;
1204                 for (i = 0; i < g_mirror_syncreqs; i++) {
1205                         sbp = disk->d_sync.ds_bios[i];
1206                         if (sbp == NULL)
1207                                 continue;
1208                         sstart = sbp->bio_offset;
1209                         send = sbp->bio_offset + sbp->bio_length;
1210                         if (rend > sstart && rstart < send)
1211                                 return (true);
1212                 }
1213         }
1214         return (false);
1215 }
1216
1217 /*
1218  * Return TRUE if the given sync request is colliding with a in-progress regular
1219  * request.
1220  */
1221 static bool
1222 g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1223 {
1224         off_t rstart, rend, sstart, send;
1225         struct bio *bp;
1226
1227         if (sc->sc_sync.ds_ndisks == 0)
1228                 return (false);
1229         sstart = sbp->bio_offset;
1230         send = sbp->bio_offset + sbp->bio_length;
1231         TAILQ_FOREACH(bp, &sc->sc_inflight, bio_queue) {
1232                 rstart = bp->bio_offset;
1233                 rend = bp->bio_offset + bp->bio_length;
1234                 if (rend > sstart && rstart < send)
1235                         return (true);
1236         }
1237         return (false);
1238 }
1239
1240 /*
1241  * Puts regular request onto delayed queue.
1242  */
1243 static void
1244 g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1245 {
1246
1247         G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1248         TAILQ_INSERT_TAIL(&sc->sc_regular_delayed, bp, bio_queue);
1249 }
1250
1251 /*
1252  * Puts synchronization request onto delayed queue.
1253  */
1254 static void
1255 g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1256 {
1257
1258         G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1259         TAILQ_INSERT_TAIL(&sc->sc_sync_delayed, bp, bio_queue);
1260 }
1261
1262 /*
1263  * Requeue delayed regular requests.
1264  */
1265 static void
1266 g_mirror_regular_release(struct g_mirror_softc *sc)
1267 {
1268         struct bio *bp;
1269
1270         if ((bp = TAILQ_FIRST(&sc->sc_regular_delayed)) == NULL)
1271                 return;
1272         if (g_mirror_sync_collision(sc, bp))
1273                 return;
1274
1275         G_MIRROR_DEBUG(2, "Requeuing regular requests after collision.");
1276         mtx_lock(&sc->sc_queue_mtx);
1277         TAILQ_CONCAT(&sc->sc_regular_delayed, &sc->sc_queue, bio_queue);
1278         TAILQ_SWAP(&sc->sc_regular_delayed, &sc->sc_queue, bio, bio_queue);
1279         mtx_unlock(&sc->sc_queue_mtx);
1280 }
1281
1282 /*
1283  * Releases delayed sync requests which don't collide anymore with regular
1284  * requests.
1285  */
1286 static void
1287 g_mirror_sync_release(struct g_mirror_softc *sc)
1288 {
1289         struct bio *bp, *bp2;
1290
1291         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed, bio_queue, bp2) {
1292                 if (g_mirror_regular_collision(sc, bp))
1293                         continue;
1294                 TAILQ_REMOVE(&sc->sc_sync_delayed, bp, bio_queue);
1295                 G_MIRROR_LOGREQ(2, bp,
1296                     "Releasing delayed synchronization request.");
1297                 g_io_request(bp, bp->bio_from);
1298         }
1299 }
1300
1301 /*
1302  * Free a synchronization request and clear its slot in the array.
1303  */
1304 static void
1305 g_mirror_sync_request_free(struct g_mirror_disk *disk, struct bio *bp)
1306 {
1307         int idx;
1308
1309         if (disk != NULL && disk->d_sync.ds_bios != NULL) {
1310                 idx = (int)(uintptr_t)bp->bio_caller1;
1311                 KASSERT(disk->d_sync.ds_bios[idx] == bp,
1312                     ("unexpected sync BIO at %p:%d", disk, idx));
1313                 disk->d_sync.ds_bios[idx] = NULL;
1314         }
1315         free(bp->bio_data, M_MIRROR);
1316         g_destroy_bio(bp);
1317 }
1318
1319 /*
1320  * Handle synchronization requests.
1321  * Every synchronization request is a two-step process: first, a read request is
1322  * sent to the mirror provider via the sync consumer. If that request completes
1323  * successfully, it is converted to a write and sent to the disk being
1324  * synchronized. If the write also completes successfully, the synchronization
1325  * offset is advanced and a new read request is submitted.
1326  */
1327 static void
1328 g_mirror_sync_request(struct g_mirror_softc *sc, struct bio *bp)
1329 {
1330         struct g_mirror_disk *disk;
1331         struct g_mirror_disk_sync *sync;
1332
1333         KASSERT((bp->bio_cmd == BIO_READ &&
1334             bp->bio_from->geom == sc->sc_sync.ds_geom) ||
1335             (bp->bio_cmd == BIO_WRITE && bp->bio_from->geom == sc->sc_geom),
1336             ("Sync BIO %p with unexpected origin", bp));
1337
1338         bp->bio_from->index--;
1339         disk = bp->bio_from->private;
1340         if (disk == NULL) {
1341                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1342                 g_topology_lock();
1343                 g_mirror_kill_consumer(sc, bp->bio_from);
1344                 g_topology_unlock();
1345                 g_mirror_sync_request_free(NULL, bp);
1346                 sx_xlock(&sc->sc_lock);
1347                 return;
1348         }
1349
1350         sync = &disk->d_sync;
1351
1352         /*
1353          * Synchronization request.
1354          */
1355         switch (bp->bio_cmd) {
1356         case BIO_READ: {
1357                 struct g_consumer *cp;
1358
1359                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_sync_request_read,
1360                     bp->bio_error);
1361
1362                 if (bp->bio_error != 0) {
1363                         G_MIRROR_LOGREQ(0, bp,
1364                             "Synchronization request failed (error=%d).",
1365                             bp->bio_error);
1366
1367                         /*
1368                          * The read error will trigger a syncid bump, so there's
1369                          * no need to do that here.
1370                          *
1371                          * The read error handling for regular requests will
1372                          * retry the read from all active mirrors before passing
1373                          * the error back up, so there's no need to retry here.
1374                          */
1375                         g_mirror_sync_request_free(disk, bp);
1376                         g_mirror_event_send(disk,
1377                             G_MIRROR_DISK_STATE_DISCONNECTED,
1378                             G_MIRROR_EVENT_DONTWAIT);
1379                         return;
1380                 }
1381                 G_MIRROR_LOGREQ(3, bp,
1382                     "Synchronization request half-finished.");
1383                 bp->bio_cmd = BIO_WRITE;
1384                 bp->bio_cflags = 0;
1385                 cp = disk->d_consumer;
1386                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1387                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1388                     cp->acr, cp->acw, cp->ace));
1389                 cp->index++;
1390                 g_io_request(bp, cp);
1391                 return;
1392         }
1393         case BIO_WRITE: {
1394                 off_t offset;
1395                 int i;
1396
1397                 KFAIL_POINT_ERROR(DEBUG_FP, g_mirror_sync_request_write,
1398                     bp->bio_error);
1399
1400                 if (bp->bio_error != 0) {
1401                         G_MIRROR_LOGREQ(0, bp,
1402                             "Synchronization request failed (error=%d).",
1403                             bp->bio_error);
1404                         g_mirror_sync_request_free(disk, bp);
1405                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1406                         g_mirror_event_send(disk,
1407                             G_MIRROR_DISK_STATE_DISCONNECTED,
1408                             G_MIRROR_EVENT_DONTWAIT);
1409                         return;
1410                 }
1411                 G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1412                 if (sync->ds_offset >= sc->sc_mediasize ||
1413                     sync->ds_consumer == NULL ||
1414                     (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1415                         /* Don't send more synchronization requests. */
1416                         sync->ds_inflight--;
1417                         g_mirror_sync_request_free(disk, bp);
1418                         if (sync->ds_inflight > 0)
1419                                 return;
1420                         if (sync->ds_consumer == NULL ||
1421                             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1422                                 return;
1423                         }
1424                         /* Disk up-to-date, activate it. */
1425                         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1426                             G_MIRROR_EVENT_DONTWAIT);
1427                         return;
1428                 }
1429
1430                 /* Send next synchronization request. */
1431                 g_mirror_sync_reinit(disk, bp, sync->ds_offset);
1432                 sync->ds_offset += bp->bio_length;
1433
1434                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1435                 sync->ds_consumer->index++;
1436
1437                 /*
1438                  * Delay the request if it is colliding with a regular request.
1439                  */
1440                 if (g_mirror_regular_collision(sc, bp))
1441                         g_mirror_sync_delay(sc, bp);
1442                 else
1443                         g_io_request(bp, sync->ds_consumer);
1444
1445                 /* Requeue delayed requests if possible. */
1446                 g_mirror_regular_release(sc);
1447
1448                 /* Find the smallest offset */
1449                 offset = sc->sc_mediasize;
1450                 for (i = 0; i < g_mirror_syncreqs; i++) {
1451                         bp = sync->ds_bios[i];
1452                         if (bp != NULL && bp->bio_offset < offset)
1453                                 offset = bp->bio_offset;
1454                 }
1455                 if (g_mirror_sync_period > 0 &&
1456                     time_uptime - sync->ds_update_ts > g_mirror_sync_period) {
1457                         sync->ds_offset_done = offset;
1458                         g_mirror_update_metadata(disk);
1459                         sync->ds_update_ts = time_uptime;
1460                 }
1461                 return;
1462         }
1463         default:
1464                 panic("Invalid I/O request %p", bp);
1465         }
1466 }
1467
1468 static void
1469 g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1470 {
1471         struct g_mirror_disk *disk;
1472         struct g_consumer *cp;
1473         struct bio *cbp;
1474
1475         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1476                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1477                         break;
1478         }
1479         if (disk == NULL) {
1480                 if (bp->bio_error == 0)
1481                         bp->bio_error = ENXIO;
1482                 g_io_deliver(bp, bp->bio_error);
1483                 return;
1484         }
1485         cbp = g_clone_bio(bp);
1486         if (cbp == NULL) {
1487                 if (bp->bio_error == 0)
1488                         bp->bio_error = ENOMEM;
1489                 g_io_deliver(bp, bp->bio_error);
1490                 return;
1491         }
1492         /*
1493          * Fill in the component buf structure.
1494          */
1495         cp = disk->d_consumer;
1496         cbp->bio_done = g_mirror_done;
1497         cbp->bio_to = cp->provider;
1498         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1499         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1500             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1501             cp->acw, cp->ace));
1502         cp->index++;
1503         g_io_request(cbp, cp);
1504 }
1505
1506 static void
1507 g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1508 {
1509         struct g_mirror_disk *disk;
1510         struct g_consumer *cp;
1511         struct bio *cbp;
1512
1513         disk = g_mirror_get_disk(sc);
1514         if (disk == NULL) {
1515                 if (bp->bio_error == 0)
1516                         bp->bio_error = ENXIO;
1517                 g_io_deliver(bp, bp->bio_error);
1518                 return;
1519         }
1520         cbp = g_clone_bio(bp);
1521         if (cbp == NULL) {
1522                 if (bp->bio_error == 0)
1523                         bp->bio_error = ENOMEM;
1524                 g_io_deliver(bp, bp->bio_error);
1525                 return;
1526         }
1527         /*
1528          * Fill in the component buf structure.
1529          */
1530         cp = disk->d_consumer;
1531         cbp->bio_done = g_mirror_done;
1532         cbp->bio_to = cp->provider;
1533         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1534         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1535             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1536             cp->acw, cp->ace));
1537         cp->index++;
1538         g_io_request(cbp, cp);
1539 }
1540
1541 #define TRACK_SIZE  (1 * 1024 * 1024)
1542 #define LOAD_SCALE      256
1543 #define ABS(x)          (((x) >= 0) ? (x) : (-(x)))
1544
1545 static void
1546 g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1547 {
1548         struct g_mirror_disk *disk, *dp;
1549         struct g_consumer *cp;
1550         struct bio *cbp;
1551         int prio, best;
1552
1553         /* Find a disk with the smallest load. */
1554         disk = NULL;
1555         best = INT_MAX;
1556         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1557                 if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1558                         continue;
1559                 prio = dp->load;
1560                 /* If disk head is precisely in position - highly prefer it. */
1561                 if (dp->d_last_offset == bp->bio_offset)
1562                         prio -= 2 * LOAD_SCALE;
1563                 else
1564                 /* If disk head is close to position - prefer it. */
1565                 if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1566                         prio -= 1 * LOAD_SCALE;
1567                 if (prio <= best) {
1568                         disk = dp;
1569                         best = prio;
1570                 }
1571         }
1572         KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1573         cbp = g_clone_bio(bp);
1574         if (cbp == NULL) {
1575                 if (bp->bio_error == 0)
1576                         bp->bio_error = ENOMEM;
1577                 g_io_deliver(bp, bp->bio_error);
1578                 return;
1579         }
1580         /*
1581          * Fill in the component buf structure.
1582          */
1583         cp = disk->d_consumer;
1584         cbp->bio_done = g_mirror_done;
1585         cbp->bio_to = cp->provider;
1586         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1587         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1588             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1589             cp->acw, cp->ace));
1590         cp->index++;
1591         /* Remember last head position */
1592         disk->d_last_offset = bp->bio_offset + bp->bio_length;
1593         /* Update loads. */
1594         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1595                 dp->load = (dp->d_consumer->index * LOAD_SCALE +
1596                     dp->load * 7) / 8;
1597         }
1598         g_io_request(cbp, cp);
1599 }
1600
1601 static void
1602 g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1603 {
1604         struct bio_queue queue;
1605         struct g_mirror_disk *disk;
1606         struct g_consumer *cp;
1607         struct bio *cbp;
1608         off_t left, mod, offset, slice;
1609         u_char *data;
1610         u_int ndisks;
1611
1612         if (bp->bio_length <= sc->sc_slice) {
1613                 g_mirror_request_round_robin(sc, bp);
1614                 return;
1615         }
1616         ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1617         slice = bp->bio_length / ndisks;
1618         mod = slice % sc->sc_provider->sectorsize;
1619         if (mod != 0)
1620                 slice += sc->sc_provider->sectorsize - mod;
1621         /*
1622          * Allocate all bios before sending any request, so we can
1623          * return ENOMEM in nice and clean way.
1624          */
1625         left = bp->bio_length;
1626         offset = bp->bio_offset;
1627         data = bp->bio_data;
1628         TAILQ_INIT(&queue);
1629         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1630                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1631                         continue;
1632                 cbp = g_clone_bio(bp);
1633                 if (cbp == NULL) {
1634                         while ((cbp = TAILQ_FIRST(&queue)) != NULL) {
1635                                 TAILQ_REMOVE(&queue, cbp, bio_queue);
1636                                 g_destroy_bio(cbp);
1637                         }
1638                         if (bp->bio_error == 0)
1639                                 bp->bio_error = ENOMEM;
1640                         g_io_deliver(bp, bp->bio_error);
1641                         return;
1642                 }
1643                 TAILQ_INSERT_TAIL(&queue, cbp, bio_queue);
1644                 cbp->bio_done = g_mirror_done;
1645                 cbp->bio_caller1 = disk;
1646                 cbp->bio_to = disk->d_consumer->provider;
1647                 cbp->bio_offset = offset;
1648                 cbp->bio_data = data;
1649                 cbp->bio_length = MIN(left, slice);
1650                 left -= cbp->bio_length;
1651                 if (left == 0)
1652                         break;
1653                 offset += cbp->bio_length;
1654                 data += cbp->bio_length;
1655         }
1656         while ((cbp = TAILQ_FIRST(&queue)) != NULL) {
1657                 TAILQ_REMOVE(&queue, cbp, bio_queue);
1658                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1659                 disk = cbp->bio_caller1;
1660                 cbp->bio_caller1 = NULL;
1661                 cp = disk->d_consumer;
1662                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1663                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1664                     cp->acr, cp->acw, cp->ace));
1665                 disk->d_consumer->index++;
1666                 g_io_request(cbp, disk->d_consumer);
1667         }
1668 }
1669
1670 static void
1671 g_mirror_register_request(struct g_mirror_softc *sc, struct bio *bp)
1672 {
1673         struct bio_queue queue;
1674         struct bio *cbp;
1675         struct g_consumer *cp;
1676         struct g_mirror_disk *disk;
1677
1678         sx_assert(&sc->sc_lock, SA_XLOCKED);
1679
1680         /*
1681          * To avoid ordering issues, if a write is deferred because of a
1682          * collision with a sync request, all I/O is deferred until that
1683          * write is initiated.
1684          */
1685         if (bp->bio_from->geom != sc->sc_sync.ds_geom &&
1686             !TAILQ_EMPTY(&sc->sc_regular_delayed)) {
1687                 g_mirror_regular_delay(sc, bp);
1688                 return;
1689         }
1690
1691         switch (bp->bio_cmd) {
1692         case BIO_READ:
1693                 switch (sc->sc_balance) {
1694                 case G_MIRROR_BALANCE_LOAD:
1695                         g_mirror_request_load(sc, bp);
1696                         break;
1697                 case G_MIRROR_BALANCE_PREFER:
1698                         g_mirror_request_prefer(sc, bp);
1699                         break;
1700                 case G_MIRROR_BALANCE_ROUND_ROBIN:
1701                         g_mirror_request_round_robin(sc, bp);
1702                         break;
1703                 case G_MIRROR_BALANCE_SPLIT:
1704                         g_mirror_request_split(sc, bp);
1705                         break;
1706                 }
1707                 return;
1708         case BIO_WRITE:
1709         case BIO_DELETE:
1710                 /*
1711                  * Delay the request if it is colliding with a synchronization
1712                  * request.
1713                  */
1714                 if (g_mirror_sync_collision(sc, bp)) {
1715                         g_mirror_regular_delay(sc, bp);
1716                         return;
1717                 }
1718
1719                 if (sc->sc_idle)
1720                         g_mirror_unidle(sc);
1721                 else
1722                         sc->sc_last_write = time_uptime;
1723
1724                 /*
1725                  * Bump syncid on first write.
1726                  */
1727                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1728                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1729                         g_mirror_bump_syncid(sc);
1730                 }
1731
1732                 /*
1733                  * Allocate all bios before sending any request, so we can
1734                  * return ENOMEM in nice and clean way.
1735                  */
1736                 TAILQ_INIT(&queue);
1737                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1738                         switch (disk->d_state) {
1739                         case G_MIRROR_DISK_STATE_ACTIVE:
1740                                 break;
1741                         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1742                                 if (bp->bio_offset >= disk->d_sync.ds_offset)
1743                                         continue;
1744                                 break;
1745                         default:
1746                                 continue;
1747                         }
1748                         if (bp->bio_cmd == BIO_DELETE &&
1749                             (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1750                                 continue;
1751                         cbp = g_clone_bio(bp);
1752                         if (cbp == NULL) {
1753                                 while ((cbp = TAILQ_FIRST(&queue)) != NULL) {
1754                                         TAILQ_REMOVE(&queue, cbp, bio_queue);
1755                                         g_destroy_bio(cbp);
1756                                 }
1757                                 if (bp->bio_error == 0)
1758                                         bp->bio_error = ENOMEM;
1759                                 g_io_deliver(bp, bp->bio_error);
1760                                 return;
1761                         }
1762                         TAILQ_INSERT_TAIL(&queue, cbp, bio_queue);
1763                         cbp->bio_done = g_mirror_done;
1764                         cp = disk->d_consumer;
1765                         cbp->bio_caller1 = cp;
1766                         cbp->bio_to = cp->provider;
1767                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1768                             ("Consumer %s not opened (r%dw%de%d).",
1769                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1770                 }
1771                 if (TAILQ_EMPTY(&queue)) {
1772                         KASSERT(bp->bio_cmd == BIO_DELETE,
1773                             ("No consumers for regular request %p", bp));
1774                         g_io_deliver(bp, EOPNOTSUPP);
1775                         return;
1776                 }
1777                 while ((cbp = TAILQ_FIRST(&queue)) != NULL) {
1778                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1779                         TAILQ_REMOVE(&queue, cbp, bio_queue);
1780                         cp = cbp->bio_caller1;
1781                         cbp->bio_caller1 = NULL;
1782                         cp->index++;
1783                         sc->sc_writes++;
1784                         g_io_request(cbp, cp);
1785                 }
1786                 /*
1787                  * Put request onto inflight queue, so we can check if new
1788                  * synchronization requests don't collide with it.
1789                  */
1790                 TAILQ_INSERT_TAIL(&sc->sc_inflight, bp, bio_queue);
1791                 return;
1792         case BIO_SPEEDUP:
1793         case BIO_FLUSH:
1794                 TAILQ_INIT(&queue);
1795                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1796                         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1797                                 continue;
1798                         cbp = g_clone_bio(bp);
1799                         if (cbp == NULL) {
1800                                 while ((cbp = TAILQ_FIRST(&queue)) != NULL) {
1801                                         TAILQ_REMOVE(&queue, cbp, bio_queue);
1802                                         g_destroy_bio(cbp);
1803                                 }
1804                                 if (bp->bio_error == 0)
1805                                         bp->bio_error = ENOMEM;
1806                                 g_io_deliver(bp, bp->bio_error);
1807                                 return;
1808                         }
1809                         TAILQ_INSERT_TAIL(&queue, cbp, bio_queue);
1810                         cbp->bio_done = g_mirror_done;
1811                         cbp->bio_caller1 = disk;
1812                         cbp->bio_to = disk->d_consumer->provider;
1813                 }
1814                 KASSERT(!TAILQ_EMPTY(&queue),
1815                     ("No consumers for regular request %p", bp));
1816                 while ((cbp = TAILQ_FIRST(&queue)) != NULL) {
1817                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1818                         TAILQ_REMOVE(&queue, cbp, bio_queue);
1819                         disk = cbp->bio_caller1;
1820                         cbp->bio_caller1 = NULL;
1821                         cp = disk->d_consumer;
1822                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1823                             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1824                             cp->acr, cp->acw, cp->ace));
1825                         cp->index++;
1826                         g_io_request(cbp, cp);
1827                 }
1828                 break;
1829         default:
1830                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1831                     bp->bio_cmd, sc->sc_name));
1832                 break;
1833         }
1834 }
1835
1836 static int
1837 g_mirror_can_destroy(struct g_mirror_softc *sc)
1838 {
1839         struct g_geom *gp;
1840         struct g_consumer *cp;
1841
1842         g_topology_assert();
1843         gp = sc->sc_geom;
1844         if (gp->softc == NULL)
1845                 return (1);
1846         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1847                 return (0);
1848         LIST_FOREACH(cp, &gp->consumer, consumer) {
1849                 if (g_mirror_is_busy(sc, cp))
1850                         return (0);
1851         }
1852         gp = sc->sc_sync.ds_geom;
1853         LIST_FOREACH(cp, &gp->consumer, consumer) {
1854                 if (g_mirror_is_busy(sc, cp))
1855                         return (0);
1856         }
1857         G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1858             sc->sc_name);
1859         return (1);
1860 }
1861
1862 static int
1863 g_mirror_try_destroy(struct g_mirror_softc *sc)
1864 {
1865
1866         if (sc->sc_rootmount != NULL) {
1867                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1868                     sc->sc_rootmount);
1869                 root_mount_rel(sc->sc_rootmount);
1870                 sc->sc_rootmount = NULL;
1871         }
1872         g_topology_lock();
1873         if (!g_mirror_can_destroy(sc)) {
1874                 g_topology_unlock();
1875                 return (0);
1876         }
1877         sc->sc_geom->softc = NULL;
1878         sc->sc_sync.ds_geom->softc = NULL;
1879         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DRAIN) != 0) {
1880                 g_topology_unlock();
1881                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1882                     &sc->sc_worker);
1883                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
1884                 sx_xunlock(&sc->sc_lock);
1885                 wakeup(&sc->sc_worker);
1886                 sc->sc_worker = NULL;
1887         } else {
1888                 g_topology_unlock();
1889                 g_mirror_destroy_device(sc);
1890         }
1891         return (1);
1892 }
1893
1894 /*
1895  * Worker thread.
1896  */
1897 static void
1898 g_mirror_worker(void *arg)
1899 {
1900         struct g_mirror_softc *sc;
1901         struct g_mirror_event *ep;
1902         struct bio *bp;
1903         int timeout;
1904
1905         sc = arg;
1906         thread_lock(curthread);
1907         sched_prio(curthread, PRIBIO);
1908         thread_unlock(curthread);
1909
1910         sx_xlock(&sc->sc_lock);
1911         for (;;) {
1912                 G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1913                 /*
1914                  * First take a look at events.
1915                  * This is important to handle events before any I/O requests.
1916                  */
1917                 ep = g_mirror_event_first(sc);
1918                 if (ep != NULL) {
1919                         g_mirror_event_remove(sc, ep);
1920                         if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1921                                 /* Update only device status. */
1922                                 G_MIRROR_DEBUG(3,
1923                                     "Running event for device %s.",
1924                                     sc->sc_name);
1925                                 ep->e_error = 0;
1926                                 g_mirror_update_device(sc, true);
1927                         } else {
1928                                 /* Update disk status. */
1929                                 G_MIRROR_DEBUG(3, "Running event for disk %s.",
1930                                      g_mirror_get_diskname(ep->e_disk));
1931                                 ep->e_error = g_mirror_update_disk(ep->e_disk,
1932                                     ep->e_state);
1933                                 if (ep->e_error == 0)
1934                                         g_mirror_update_device(sc, false);
1935                         }
1936                         if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1937                                 KASSERT(ep->e_error == 0,
1938                                     ("Error cannot be handled."));
1939                                 g_mirror_event_free(ep);
1940                         } else {
1941                                 ep->e_flags |= G_MIRROR_EVENT_DONE;
1942                                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1943                                     ep);
1944                                 mtx_lock(&sc->sc_events_mtx);
1945                                 wakeup(ep);
1946                                 mtx_unlock(&sc->sc_events_mtx);
1947                         }
1948                         if ((sc->sc_flags &
1949                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1950                                 if (g_mirror_try_destroy(sc)) {
1951                                         curthread->td_pflags &= ~TDP_GEOM;
1952                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1953                                         kproc_exit(0);
1954                                 }
1955                         }
1956                         G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1957                         continue;
1958                 }
1959
1960                 /*
1961                  * Check if we can mark array as CLEAN and if we can't take
1962                  * how much seconds should we wait.
1963                  */
1964                 timeout = g_mirror_idle(sc, -1);
1965
1966                 /*
1967                  * Handle I/O requests.
1968                  */
1969                 mtx_lock(&sc->sc_queue_mtx);
1970                 bp = TAILQ_FIRST(&sc->sc_queue);
1971                 if (bp != NULL)
1972                         TAILQ_REMOVE(&sc->sc_queue, bp, bio_queue);
1973                 else {
1974                         if ((sc->sc_flags &
1975                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1976                                 mtx_unlock(&sc->sc_queue_mtx);
1977                                 if (g_mirror_try_destroy(sc)) {
1978                                         curthread->td_pflags &= ~TDP_GEOM;
1979                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1980                                         kproc_exit(0);
1981                                 }
1982                                 mtx_lock(&sc->sc_queue_mtx);
1983                                 if (!TAILQ_EMPTY(&sc->sc_queue)) {
1984                                         mtx_unlock(&sc->sc_queue_mtx);
1985                                         continue;
1986                                 }
1987                         }
1988                         if (g_mirror_event_first(sc) != NULL) {
1989                                 mtx_unlock(&sc->sc_queue_mtx);
1990                                 continue;
1991                         }
1992                         sx_xunlock(&sc->sc_lock);
1993                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1994                             timeout * hz);
1995                         sx_xlock(&sc->sc_lock);
1996                         G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1997                         continue;
1998                 }
1999                 mtx_unlock(&sc->sc_queue_mtx);
2000
2001                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
2002                     (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
2003                         /*
2004                          * Handle completion of the first half (the read) of a
2005                          * block synchronization operation.
2006                          */
2007                         g_mirror_sync_request(sc, bp);
2008                 } else if (bp->bio_to != sc->sc_provider) {
2009                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
2010                                 /*
2011                                  * Handle completion of a regular I/O request.
2012                                  */
2013                                 g_mirror_regular_request(sc, bp);
2014                         else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
2015                                 /*
2016                                  * Handle completion of the second half (the
2017                                  * write) of a block synchronization operation.
2018                                  */
2019                                 g_mirror_sync_request(sc, bp);
2020                         else {
2021                                 KASSERT(0,
2022                                     ("Invalid request cflags=0x%hx to=%s.",
2023                                     bp->bio_cflags, bp->bio_to->name));
2024                         }
2025                 } else {
2026                         /*
2027                          * Initiate an I/O request.
2028                          */
2029                         g_mirror_register_request(sc, bp);
2030                 }
2031                 G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
2032         }
2033 }
2034
2035 static void
2036 g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
2037 {
2038
2039         sx_assert(&sc->sc_lock, SX_LOCKED);
2040
2041         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
2042                 return;
2043         if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2044                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as dirty.",
2045                     g_mirror_get_diskname(disk), sc->sc_name);
2046                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
2047         } else if (sc->sc_idle &&
2048             (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2049                 G_MIRROR_DEBUG(2, "Disk %s (device %s) marked as clean.",
2050                     g_mirror_get_diskname(disk), sc->sc_name);
2051                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2052         }
2053 }
2054
2055 static void
2056 g_mirror_sync_reinit(const struct g_mirror_disk *disk, struct bio *bp,
2057     off_t offset)
2058 {
2059         void *data;
2060         int idx;
2061
2062         data = bp->bio_data;
2063         idx = (int)(uintptr_t)bp->bio_caller1;
2064         g_reset_bio(bp);
2065
2066         bp->bio_cmd = BIO_READ;
2067         bp->bio_data = data;
2068         bp->bio_done = g_mirror_sync_done;
2069         bp->bio_from = disk->d_sync.ds_consumer;
2070         bp->bio_to = disk->d_softc->sc_provider;
2071         bp->bio_caller1 = (void *)(uintptr_t)idx;
2072         bp->bio_offset = offset;
2073         bp->bio_length = MIN(MAXPHYS,
2074             disk->d_softc->sc_mediasize - bp->bio_offset);
2075 }
2076
2077 static void
2078 g_mirror_sync_start(struct g_mirror_disk *disk)
2079 {
2080         struct g_mirror_softc *sc;
2081         struct g_mirror_disk_sync *sync;
2082         struct g_consumer *cp;
2083         struct bio *bp;
2084         int error, i;
2085
2086         g_topology_assert_not();
2087         sc = disk->d_softc;
2088         sync = &disk->d_sync;
2089         sx_assert(&sc->sc_lock, SX_LOCKED);
2090
2091         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2092             ("Disk %s is not marked for synchronization.",
2093             g_mirror_get_diskname(disk)));
2094         KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2095             ("Device not in RUNNING state (%s, %u).", sc->sc_name,
2096             sc->sc_state));
2097
2098         sx_xunlock(&sc->sc_lock);
2099         g_topology_lock();
2100         cp = g_new_consumer(sc->sc_sync.ds_geom);
2101         cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
2102         error = g_attach(cp, sc->sc_provider);
2103         KASSERT(error == 0,
2104             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
2105         error = g_access(cp, 1, 0, 0);
2106         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
2107         g_topology_unlock();
2108         sx_xlock(&sc->sc_lock);
2109
2110         G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
2111             g_mirror_get_diskname(disk));
2112         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
2113                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
2114         KASSERT(sync->ds_consumer == NULL,
2115             ("Sync consumer already exists (device=%s, disk=%s).",
2116             sc->sc_name, g_mirror_get_diskname(disk)));
2117
2118         sync->ds_consumer = cp;
2119         sync->ds_consumer->private = disk;
2120         sync->ds_consumer->index = 0;
2121
2122         /*
2123          * Allocate memory for synchronization bios and initialize them.
2124          */
2125         sync->ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
2126             M_MIRROR, M_WAITOK);
2127         for (i = 0; i < g_mirror_syncreqs; i++) {
2128                 bp = g_alloc_bio();
2129                 sync->ds_bios[i] = bp;
2130
2131                 bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
2132                 bp->bio_caller1 = (void *)(uintptr_t)i;
2133                 g_mirror_sync_reinit(disk, bp, sync->ds_offset);
2134                 sync->ds_offset += bp->bio_length;
2135         }
2136
2137         /* Increase the number of disks in SYNCHRONIZING state. */
2138         sc->sc_sync.ds_ndisks++;
2139         /* Set the number of in-flight synchronization requests. */
2140         sync->ds_inflight = g_mirror_syncreqs;
2141
2142         /*
2143          * Fire off first synchronization requests.
2144          */
2145         for (i = 0; i < g_mirror_syncreqs; i++) {
2146                 bp = sync->ds_bios[i];
2147                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
2148                 sync->ds_consumer->index++;
2149                 /*
2150                  * Delay the request if it is colliding with a regular request.
2151                  */
2152                 if (g_mirror_regular_collision(sc, bp))
2153                         g_mirror_sync_delay(sc, bp);
2154                 else
2155                         g_io_request(bp, sync->ds_consumer);
2156         }
2157 }
2158
2159 /*
2160  * Stop synchronization process.
2161  * type: 0 - synchronization finished
2162  *       1 - synchronization stopped
2163  */
2164 static void
2165 g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2166 {
2167         struct g_mirror_softc *sc;
2168         struct g_consumer *cp;
2169
2170         g_topology_assert_not();
2171         sc = disk->d_softc;
2172         sx_assert(&sc->sc_lock, SX_LOCKED);
2173
2174         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2175             ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2176             g_mirror_disk_state2str(disk->d_state)));
2177         if (disk->d_sync.ds_consumer == NULL)
2178                 return;
2179
2180         if (type == 0) {
2181                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2182                     sc->sc_name, g_mirror_get_diskname(disk));
2183         } else /* if (type == 1) */ {
2184                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2185                     sc->sc_name, g_mirror_get_diskname(disk));
2186         }
2187         g_mirror_regular_release(sc);
2188         free(disk->d_sync.ds_bios, M_MIRROR);
2189         disk->d_sync.ds_bios = NULL;
2190         cp = disk->d_sync.ds_consumer;
2191         disk->d_sync.ds_consumer = NULL;
2192         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2193         sc->sc_sync.ds_ndisks--;
2194         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2195         g_topology_lock();
2196         g_mirror_kill_consumer(sc, cp);
2197         g_topology_unlock();
2198         sx_xlock(&sc->sc_lock);
2199 }
2200
2201 static void
2202 g_mirror_launch_provider(struct g_mirror_softc *sc)
2203 {
2204         struct g_mirror_disk *disk;
2205         struct g_provider *pp, *dp;
2206
2207         sx_assert(&sc->sc_lock, SX_LOCKED);
2208
2209         g_topology_lock();
2210         pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2211         pp->flags |= G_PF_DIRECT_RECEIVE;
2212         pp->mediasize = sc->sc_mediasize;
2213         pp->sectorsize = sc->sc_sectorsize;
2214         pp->stripesize = 0;
2215         pp->stripeoffset = 0;
2216
2217         /* Splitting of unmapped BIO's could work but isn't implemented now */
2218         if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2219                 pp->flags |= G_PF_ACCEPT_UNMAPPED;
2220
2221         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2222                 if (disk->d_consumer && disk->d_consumer->provider) {
2223                         dp = disk->d_consumer->provider;
2224                         if (dp->stripesize > pp->stripesize) {
2225                                 pp->stripesize = dp->stripesize;
2226                                 pp->stripeoffset = dp->stripeoffset;
2227                         }
2228                         /* A provider underneath us doesn't support unmapped */
2229                         if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2230                                 G_MIRROR_DEBUG(0, "Cancelling unmapped "
2231                                     "because of %s.", dp->name);
2232                                 pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2233                         }
2234                 }
2235         }
2236         pp->private = sc;
2237         sc->sc_refcnt++;
2238         sc->sc_provider = pp;
2239         g_error_provider(pp, 0);
2240         g_topology_unlock();
2241         G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2242             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2243         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2244                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2245                         g_mirror_sync_start(disk);
2246         }
2247 }
2248
2249 static void
2250 g_mirror_destroy_provider(struct g_mirror_softc *sc)
2251 {
2252         struct g_mirror_disk *disk;
2253         struct bio *bp;
2254
2255         g_topology_assert_not();
2256         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2257             sc->sc_name));
2258
2259         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2260                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2261                         g_mirror_sync_stop(disk, 1);
2262         }
2263
2264         g_topology_lock();
2265         g_error_provider(sc->sc_provider, ENXIO);
2266         mtx_lock(&sc->sc_queue_mtx);
2267         while ((bp = TAILQ_FIRST(&sc->sc_queue)) != NULL) {
2268                 TAILQ_REMOVE(&sc->sc_queue, bp, bio_queue);
2269                 /*
2270                  * Abort any pending I/O that wasn't generated by us.
2271                  * Synchronization requests and requests destined for individual
2272                  * mirror components can be destroyed immediately.
2273                  */
2274                 if (bp->bio_to == sc->sc_provider &&
2275                     bp->bio_from->geom != sc->sc_sync.ds_geom) {
2276                         g_io_deliver(bp, ENXIO);
2277                 } else {
2278                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
2279                                 free(bp->bio_data, M_MIRROR);
2280                         g_destroy_bio(bp);
2281                 }
2282         }
2283         mtx_unlock(&sc->sc_queue_mtx);
2284         g_wither_provider(sc->sc_provider, ENXIO);
2285         sc->sc_provider = NULL;
2286         G_MIRROR_DEBUG(0, "Device %s: provider destroyed.", sc->sc_name);
2287         g_topology_unlock();
2288 }
2289
2290 static void
2291 g_mirror_go(void *arg)
2292 {
2293         struct g_mirror_softc *sc;
2294
2295         sc = arg;
2296         G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2297         g_mirror_event_send(sc, 0,
2298             G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2299 }
2300
2301 static u_int
2302 g_mirror_determine_state(struct g_mirror_disk *disk)
2303 {
2304         struct g_mirror_softc *sc;
2305         u_int state;
2306
2307         sc = disk->d_softc;
2308         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2309                 if ((disk->d_flags &
2310                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0 &&
2311                     (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 ||
2312                      (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0)) {
2313                         /* Disk does not need synchronization. */
2314                         state = G_MIRROR_DISK_STATE_ACTIVE;
2315                 } else {
2316                         if ((sc->sc_flags &
2317                              G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2318                             (disk->d_flags &
2319                              G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2320                                 /*
2321                                  * We can start synchronization from
2322                                  * the stored offset.
2323                                  */
2324                                 state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2325                         } else {
2326                                 state = G_MIRROR_DISK_STATE_STALE;
2327                         }
2328                 }
2329         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2330                 /*
2331                  * Reset all synchronization data for this disk,
2332                  * because if it even was synchronized, it was
2333                  * synchronized to disks with different syncid.
2334                  */
2335                 disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2336                 disk->d_sync.ds_offset = 0;
2337                 disk->d_sync.ds_offset_done = 0;
2338                 disk->d_sync.ds_syncid = sc->sc_syncid;
2339                 if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2340                     (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2341                         state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2342                 } else {
2343                         state = G_MIRROR_DISK_STATE_STALE;
2344                 }
2345         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2346                 /*
2347                  * Not good, NOT GOOD!
2348                  * It means that mirror was started on stale disks
2349                  * and more fresh disk just arrive.
2350                  * If there were writes, mirror is broken, sorry.
2351                  * I think the best choice here is don't touch
2352                  * this disk and inform the user loudly.
2353                  */
2354                 G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2355                     "disk (%s) arrives!! It will not be connected to the "
2356                     "running device.", sc->sc_name,
2357                     g_mirror_get_diskname(disk));
2358                 g_mirror_destroy_disk(disk);
2359                 state = G_MIRROR_DISK_STATE_NONE;
2360                 /* Return immediately, because disk was destroyed. */
2361                 return (state);
2362         }
2363         G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2364             g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2365         return (state);
2366 }
2367
2368 /*
2369  * Update device state.
2370  */
2371 static void
2372 g_mirror_update_device(struct g_mirror_softc *sc, bool force)
2373 {
2374         struct g_mirror_disk *disk;
2375         u_int state;
2376
2377         sx_assert(&sc->sc_lock, SX_XLOCKED);
2378
2379         switch (sc->sc_state) {
2380         case G_MIRROR_DEVICE_STATE_STARTING:
2381             {
2382                 struct g_mirror_disk *pdisk, *tdisk;
2383                 const char *mismatch;
2384                 uintmax_t found, newest;
2385                 u_int dirty, ndisks;
2386
2387                 /* Pre-flight checks */
2388                 LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2389                         /*
2390                          * Confirm we already detected the newest genid.
2391                          */
2392                         KASSERT(sc->sc_genid >= disk->d_genid,
2393                             ("%s: found newer genid %u (sc:%p had %u).", __func__,
2394                             disk->d_genid, sc, sc->sc_genid));
2395
2396                         /* Kick out any previously tasted stale components. */
2397                         if (disk->d_genid < sc->sc_genid) {
2398                                 G_MIRROR_DEBUG(0, "Stale 'genid' field on %s "
2399                                     "(device %s) (component=%u latest=%u), skipping.",
2400                                     g_mirror_get_diskname(disk), sc->sc_name,
2401                                     disk->d_genid, sc->sc_genid);
2402                                 g_mirror_destroy_disk(disk);
2403                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2404                                 continue;
2405                         }
2406
2407                         /*
2408                          * Confirm we already detected the newest syncid.
2409                          */
2410                         KASSERT(sc->sc_syncid >= disk->d_sync.ds_syncid,
2411                             ("%s: found newer syncid %u (sc:%p had %u).",
2412                              __func__, disk->d_sync.ds_syncid, sc,
2413                              sc->sc_syncid));
2414
2415 #define DETECT_MISMATCH(field, name) \
2416                         if (mismatch == NULL &&                                 \
2417                             disk->d_init_ ## field != sc->sc_ ## field) {       \
2418                                 mismatch = name;                                \
2419                                 found = (intmax_t)disk->d_init_ ## field;       \
2420                                 newest = (intmax_t)sc->sc_ ## field;            \
2421                         }
2422                         mismatch = NULL;
2423                         DETECT_MISMATCH(ndisks, "md_all");
2424                         DETECT_MISMATCH(balance, "md_balance");
2425                         DETECT_MISMATCH(slice, "md_slice");
2426                         DETECT_MISMATCH(mediasize, "md_mediasize");
2427 #undef DETECT_MISMATCH
2428                         if (mismatch != NULL) {
2429                                 G_MIRROR_DEBUG(0, "Found a mismatching '%s' "
2430                                     "field on %s (device %s) (found=%ju "
2431                                     "newest=%ju).", mismatch,
2432                                     g_mirror_get_diskname(disk), sc->sc_name,
2433                                     found, newest);
2434                                 g_mirror_destroy_disk(disk);
2435                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2436                                 continue;
2437                         }
2438                 }
2439
2440                 KASSERT(sc->sc_provider == NULL,
2441                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2442                 /*
2443                  * Are we ready? If the timeout (force is true) has expired, and
2444                  * any disks are present, then yes. If we're permitted to launch
2445                  * before the timeout has expired and the expected number of
2446                  * current-generation mirror disks have been tasted, then yes.
2447                  */
2448                 ndisks = g_mirror_ndisks(sc, -1);
2449                 if ((force && ndisks > 0) ||
2450                     (g_launch_mirror_before_timeout && ndisks == sc->sc_ndisks)) {
2451                         ;
2452                 } else if (ndisks == 0) {
2453                         /*
2454                          * Disks went down in starting phase, so destroy
2455                          * device.
2456                          */
2457                         callout_drain(&sc->sc_callout);
2458                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2459                         G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2460                             sc->sc_rootmount);
2461                         root_mount_rel(sc->sc_rootmount);
2462                         sc->sc_rootmount = NULL;
2463                         return;
2464                 } else {
2465                         return;
2466                 }
2467
2468                 /*
2469                  * Activate all disks with the biggest syncid.
2470                  */
2471                 if (force) {
2472                         /*
2473                          * If 'force' is true, we have been called due to
2474                          * timeout, so don't bother canceling timeout.
2475                          */
2476                         ndisks = 0;
2477                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2478                                 if ((disk->d_flags &
2479                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2480                                         ndisks++;
2481                                 }
2482                         }
2483                         if (ndisks == 0) {
2484                                 /* No valid disks found, destroy device. */
2485                                 sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2486                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2487                                     __LINE__, sc->sc_rootmount);
2488                                 root_mount_rel(sc->sc_rootmount);
2489                                 sc->sc_rootmount = NULL;
2490                                 return;
2491                         }
2492                 } else {
2493                         /* Cancel timeout. */
2494                         callout_drain(&sc->sc_callout);
2495                 }
2496
2497                 /*
2498                  * Here we need to look for dirty disks and if all disks
2499                  * with the biggest syncid are dirty, we have to choose
2500                  * one with the biggest priority and rebuild the rest.
2501                  */
2502                 /*
2503                  * Find the number of dirty disks with the biggest syncid.
2504                  * Find the number of disks with the biggest syncid.
2505                  * While here, find a disk with the biggest priority.
2506                  */
2507                 dirty = ndisks = 0;
2508                 pdisk = NULL;
2509                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2510                         if (disk->d_sync.ds_syncid != sc->sc_syncid)
2511                                 continue;
2512                         if ((disk->d_flags &
2513                             G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2514                                 continue;
2515                         }
2516                         ndisks++;
2517                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2518                                 dirty++;
2519                                 if (pdisk == NULL ||
2520                                     pdisk->d_priority < disk->d_priority) {
2521                                         pdisk = disk;
2522                                 }
2523                         }
2524                 }
2525                 if (dirty == 0) {
2526                         /* No dirty disks at all, great. */
2527                 } else if (dirty == ndisks) {
2528                         /*
2529                          * Force synchronization for all dirty disks except one
2530                          * with the biggest priority.
2531                          */
2532                         KASSERT(pdisk != NULL, ("pdisk == NULL"));
2533                         G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2534                             "master disk for synchronization.",
2535                             g_mirror_get_diskname(pdisk), sc->sc_name);
2536                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2537                                 if (disk->d_sync.ds_syncid != sc->sc_syncid)
2538                                         continue;
2539                                 if ((disk->d_flags &
2540                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2541                                         continue;
2542                                 }
2543                                 KASSERT((disk->d_flags &
2544                                     G_MIRROR_DISK_FLAG_DIRTY) != 0,
2545                                     ("Disk %s isn't marked as dirty.",
2546                                     g_mirror_get_diskname(disk)));
2547                                 /* Skip the disk with the biggest priority. */
2548                                 if (disk == pdisk)
2549                                         continue;
2550                                 disk->d_sync.ds_syncid = 0;
2551                         }
2552                 } else if (dirty < ndisks) {
2553                         /*
2554                          * Force synchronization for all dirty disks.
2555                          * We have some non-dirty disks.
2556                          */
2557                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2558                                 if (disk->d_sync.ds_syncid != sc->sc_syncid)
2559                                         continue;
2560                                 if ((disk->d_flags &
2561                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2562                                         continue;
2563                                 }
2564                                 if ((disk->d_flags &
2565                                     G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2566                                         continue;
2567                                 }
2568                                 disk->d_sync.ds_syncid = 0;
2569                         }
2570                 }
2571
2572                 /* Reset hint. */
2573                 sc->sc_hint = NULL;
2574                 if (force) {
2575                         /* Remember to bump syncid on first write. */
2576                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2577                 }
2578                 state = G_MIRROR_DEVICE_STATE_RUNNING;
2579                 G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2580                     sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2581                     g_mirror_device_state2str(state));
2582                 sc->sc_state = state;
2583                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2584                         state = g_mirror_determine_state(disk);
2585                         g_mirror_event_send(disk, state,
2586                             G_MIRROR_EVENT_DONTWAIT);
2587                         if (state == G_MIRROR_DISK_STATE_STALE)
2588                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2589                 }
2590                 break;
2591             }
2592         case G_MIRROR_DEVICE_STATE_RUNNING:
2593                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2594                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2595                         /*
2596                          * No usable disks, so destroy the device.
2597                          */
2598                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2599                         break;
2600                 } else if (g_mirror_ndisks(sc,
2601                     G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2602                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2603                         /*
2604                          * We have active disks, launch provider if it doesn't
2605                          * exist.
2606                          */
2607                         if (sc->sc_provider == NULL)
2608                                 g_mirror_launch_provider(sc);
2609                         if (sc->sc_rootmount != NULL) {
2610                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2611                                     __LINE__, sc->sc_rootmount);
2612                                 root_mount_rel(sc->sc_rootmount);
2613                                 sc->sc_rootmount = NULL;
2614                         }
2615                 }
2616                 /*
2617                  * Genid should be bumped immediately, so do it here.
2618                  */
2619                 if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2620                         sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2621                         g_mirror_bump_genid(sc);
2622                 }
2623                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID_NOW) != 0) {
2624                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID_NOW;
2625                         g_mirror_bump_syncid(sc);
2626                 }
2627                 break;
2628         default:
2629                 KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2630                     sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2631                 break;
2632         }
2633 }
2634
2635 /*
2636  * Update disk state and device state if needed.
2637  */
2638 #define DISK_STATE_CHANGED()    G_MIRROR_DEBUG(1,                       \
2639         "Disk %s state changed from %s to %s (device %s).",             \
2640         g_mirror_get_diskname(disk),                                    \
2641         g_mirror_disk_state2str(disk->d_state),                         \
2642         g_mirror_disk_state2str(state), sc->sc_name)
2643 static int
2644 g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2645 {
2646         struct g_mirror_softc *sc;
2647
2648         sc = disk->d_softc;
2649         sx_assert(&sc->sc_lock, SX_XLOCKED);
2650
2651 again:
2652         G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2653             g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2654             g_mirror_disk_state2str(state));
2655         switch (state) {
2656         case G_MIRROR_DISK_STATE_NEW:
2657                 /*
2658                  * Possible scenarios:
2659                  * 1. New disk arrive.
2660                  */
2661                 /* Previous state should be NONE. */
2662                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2663                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2664                     g_mirror_disk_state2str(disk->d_state)));
2665                 DISK_STATE_CHANGED();
2666
2667                 disk->d_state = state;
2668                 g_topology_lock();
2669                 if (LIST_EMPTY(&sc->sc_disks))
2670                         LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2671                 else {
2672                         struct g_mirror_disk *dp;
2673
2674                         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2675                                 if (disk->d_priority >= dp->d_priority) {
2676                                         LIST_INSERT_BEFORE(dp, disk, d_next);
2677                                         dp = NULL;
2678                                         break;
2679                                 }
2680                                 if (LIST_NEXT(dp, d_next) == NULL)
2681                                         break;
2682                         }
2683                         if (dp != NULL)
2684                                 LIST_INSERT_AFTER(dp, disk, d_next);
2685                 }
2686                 g_topology_unlock();
2687                 G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2688                     sc->sc_name, g_mirror_get_diskname(disk));
2689                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2690                         break;
2691                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2692                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2693                     g_mirror_device_state2str(sc->sc_state),
2694                     g_mirror_get_diskname(disk),
2695                     g_mirror_disk_state2str(disk->d_state)));
2696                 state = g_mirror_determine_state(disk);
2697                 if (state != G_MIRROR_DISK_STATE_NONE)
2698                         goto again;
2699                 break;
2700         case G_MIRROR_DISK_STATE_ACTIVE:
2701                 /*
2702                  * Possible scenarios:
2703                  * 1. New disk does not need synchronization.
2704                  * 2. Synchronization process finished successfully.
2705                  */
2706                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2707                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2708                     g_mirror_device_state2str(sc->sc_state),
2709                     g_mirror_get_diskname(disk),
2710                     g_mirror_disk_state2str(disk->d_state)));
2711                 /* Previous state should be NEW or SYNCHRONIZING. */
2712                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2713                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2714                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2715                     g_mirror_disk_state2str(disk->d_state)));
2716                 DISK_STATE_CHANGED();
2717
2718                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2719                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2720                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2721                         g_mirror_sync_stop(disk, 0);
2722                 }
2723                 disk->d_state = state;
2724                 disk->d_sync.ds_offset = 0;
2725                 disk->d_sync.ds_offset_done = 0;
2726                 g_mirror_update_idle(sc, disk);
2727                 g_mirror_update_metadata(disk);
2728                 G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2729                     sc->sc_name, g_mirror_get_diskname(disk));
2730                 break;
2731         case G_MIRROR_DISK_STATE_STALE:
2732                 /*
2733                  * Possible scenarios:
2734                  * 1. Stale disk was connected.
2735                  */
2736                 /* Previous state should be NEW. */
2737                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2738                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2739                     g_mirror_disk_state2str(disk->d_state)));
2740                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2741                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2742                     g_mirror_device_state2str(sc->sc_state),
2743                     g_mirror_get_diskname(disk),
2744                     g_mirror_disk_state2str(disk->d_state)));
2745                 /*
2746                  * STALE state is only possible if device is marked
2747                  * NOAUTOSYNC.
2748                  */
2749                 KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2750                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2751                     g_mirror_device_state2str(sc->sc_state),
2752                     g_mirror_get_diskname(disk),
2753                     g_mirror_disk_state2str(disk->d_state)));
2754                 DISK_STATE_CHANGED();
2755
2756                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2757                 disk->d_state = state;
2758                 g_mirror_update_metadata(disk);
2759                 G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2760                     sc->sc_name, g_mirror_get_diskname(disk));
2761                 break;
2762         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2763                 /*
2764                  * Possible scenarios:
2765                  * 1. Disk which needs synchronization was connected.
2766                  */
2767                 /* Previous state should be NEW. */
2768                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2769                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2770                     g_mirror_disk_state2str(disk->d_state)));
2771                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2772                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2773                     g_mirror_device_state2str(sc->sc_state),
2774                     g_mirror_get_diskname(disk),
2775                     g_mirror_disk_state2str(disk->d_state)));
2776                 DISK_STATE_CHANGED();
2777
2778                 if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2779                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2780                 disk->d_state = state;
2781                 if (sc->sc_provider != NULL) {
2782                         g_mirror_sync_start(disk);
2783                         g_mirror_update_metadata(disk);
2784                 }
2785                 break;
2786         case G_MIRROR_DISK_STATE_DISCONNECTED:
2787                 /*
2788                  * Possible scenarios:
2789                  * 1. Device wasn't running yet, but disk disappear.
2790                  * 2. Disk was active and disapppear.
2791                  * 3. Disk disappear during synchronization process.
2792                  */
2793                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2794                         /*
2795                          * Previous state should be ACTIVE, STALE or
2796                          * SYNCHRONIZING.
2797                          */
2798                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2799                             disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2800                             disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2801                             ("Wrong disk state (%s, %s).",
2802                             g_mirror_get_diskname(disk),
2803                             g_mirror_disk_state2str(disk->d_state)));
2804                 } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2805                         /* Previous state should be NEW. */
2806                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2807                             ("Wrong disk state (%s, %s).",
2808                             g_mirror_get_diskname(disk),
2809                             g_mirror_disk_state2str(disk->d_state)));
2810                         /*
2811                          * Reset bumping syncid if disk disappeared in STARTING
2812                          * state.
2813                          */
2814                         if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2815                                 sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2816 #ifdef  INVARIANTS
2817                 } else {
2818                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2819                             sc->sc_name,
2820                             g_mirror_device_state2str(sc->sc_state),
2821                             g_mirror_get_diskname(disk),
2822                             g_mirror_disk_state2str(disk->d_state)));
2823 #endif
2824                 }
2825                 DISK_STATE_CHANGED();
2826                 G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2827                     sc->sc_name, g_mirror_get_diskname(disk));
2828
2829                 g_mirror_destroy_disk(disk);
2830                 break;
2831         case G_MIRROR_DISK_STATE_DESTROY:
2832             {
2833                 int error;
2834
2835                 error = g_mirror_clear_metadata(disk);
2836                 if (error != 0) {
2837                         G_MIRROR_DEBUG(0,
2838                             "Device %s: failed to clear metadata on %s: %d.",
2839                             sc->sc_name, g_mirror_get_diskname(disk), error);
2840                         break;
2841                 }
2842                 DISK_STATE_CHANGED();
2843                 G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2844                     sc->sc_name, g_mirror_get_diskname(disk));
2845
2846                 g_mirror_destroy_disk(disk);
2847                 sc->sc_ndisks--;
2848                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2849                         g_mirror_update_metadata(disk);
2850                 }
2851                 break;
2852             }
2853         default:
2854                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2855                 break;
2856         }
2857         return (0);
2858 }
2859 #undef  DISK_STATE_CHANGED
2860
2861 int
2862 g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2863 {
2864         struct g_provider *pp;
2865         u_char *buf;
2866         int error;
2867
2868         g_topology_assert();
2869
2870         error = g_access(cp, 1, 0, 0);
2871         if (error != 0)
2872                 return (error);
2873         pp = cp->provider;
2874         g_topology_unlock();
2875         /* Metadata are stored on last sector. */
2876         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2877             &error);
2878         g_topology_lock();
2879         g_access(cp, -1, 0, 0);
2880         if (buf == NULL) {
2881                 G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2882                     cp->provider->name, error);
2883                 return (error);
2884         }
2885
2886         /* Decode metadata. */
2887         error = mirror_metadata_decode(buf, md);
2888         g_free(buf);
2889         if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2890                 return (EINVAL);
2891         if (md->md_version > G_MIRROR_VERSION) {
2892                 G_MIRROR_DEBUG(0,
2893                     "Kernel module is too old to handle metadata from %s.",
2894                     cp->provider->name);
2895                 return (EINVAL);
2896         }
2897         if (error != 0) {
2898                 G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2899                     cp->provider->name);
2900                 return (error);
2901         }
2902
2903         return (0);
2904 }
2905
2906 static int
2907 g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2908     struct g_mirror_metadata *md)
2909 {
2910
2911         G_MIRROR_DEBUG(2, "%s: md_did 0x%u disk %s device %s md_all 0x%x "
2912             "sc_ndisks 0x%x md_slice 0x%x sc_slice 0x%x md_balance 0x%x "
2913             "sc_balance 0x%x sc_mediasize 0x%jx pp_mediasize 0x%jx "
2914             "md_sectorsize 0x%x sc_sectorsize 0x%x md_mflags 0x%jx "
2915             "md_dflags 0x%jx md_syncid 0x%x md_genid 0x%x md_priority 0x%x "
2916             "sc_state 0x%x.",
2917             __func__, md->md_did, pp->name, sc->sc_name, md->md_all,
2918             sc->sc_ndisks, md->md_slice, sc->sc_slice, md->md_balance,
2919             sc->sc_balance, (uintmax_t)sc->sc_mediasize,
2920             (uintmax_t)pp->mediasize, md->md_sectorsize, sc->sc_sectorsize,
2921             (uintmax_t)md->md_mflags, (uintmax_t)md->md_dflags, md->md_syncid,
2922             md->md_genid, md->md_priority, sc->sc_state);
2923
2924         if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2925                 G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2926                     pp->name, md->md_did);
2927                 return (EEXIST);
2928         }
2929         if (sc->sc_mediasize > pp->mediasize) {
2930                 G_MIRROR_DEBUG(1,
2931                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2932                     sc->sc_name);
2933                 return (EINVAL);
2934         }
2935         if (md->md_sectorsize != sc->sc_sectorsize) {
2936                 G_MIRROR_DEBUG(1,
2937                     "Invalid '%s' field on disk %s (device %s), skipping.",
2938                     "md_sectorsize", pp->name, sc->sc_name);
2939                 return (EINVAL);
2940         }
2941         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2942                 G_MIRROR_DEBUG(1,
2943                     "Invalid sector size of disk %s (device %s), skipping.",
2944                     pp->name, sc->sc_name);
2945                 return (EINVAL);
2946         }
2947         if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2948                 G_MIRROR_DEBUG(1,
2949                     "Invalid device flags on disk %s (device %s), skipping.",
2950                     pp->name, sc->sc_name);
2951                 return (EINVAL);
2952         }
2953         if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2954                 G_MIRROR_DEBUG(1,
2955                     "Invalid disk flags on disk %s (device %s), skipping.",
2956                     pp->name, sc->sc_name);
2957                 return (EINVAL);
2958         }
2959         return (0);
2960 }
2961
2962 int
2963 g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2964     struct g_mirror_metadata *md)
2965 {
2966         struct g_mirror_disk *disk;
2967         int error;
2968
2969         g_topology_assert_not();
2970         G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2971
2972         error = g_mirror_check_metadata(sc, pp, md);
2973         if (error != 0)
2974                 return (error);
2975
2976         if (md->md_genid < sc->sc_genid) {
2977                 G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2978                     pp->name, sc->sc_name);
2979                 return (EINVAL);
2980         }
2981
2982         /*
2983          * If the component disk we're tasting has newer metadata than the
2984          * STARTING gmirror device, refresh the device from the component.
2985          */
2986         error = g_mirror_refresh_device(sc, pp, md);
2987         if (error != 0)
2988                 return (error);
2989
2990         disk = g_mirror_init_disk(sc, pp, md, &error);
2991         if (disk == NULL)
2992                 return (error);
2993         error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2994             G_MIRROR_EVENT_WAIT);
2995         if (error != 0)
2996                 return (error);
2997         if (md->md_version < G_MIRROR_VERSION) {
2998                 G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2999                     pp->name, md->md_version, G_MIRROR_VERSION);
3000                 g_mirror_update_metadata(disk);
3001         }
3002         return (0);
3003 }
3004
3005 static void
3006 g_mirror_destroy_delayed(void *arg, int flag)
3007 {
3008         struct g_mirror_softc *sc;
3009         int error;
3010
3011         if (flag == EV_CANCEL) {
3012                 G_MIRROR_DEBUG(1, "Destroying canceled.");
3013                 return;
3014         }
3015         sc = arg;
3016         g_topology_unlock();
3017         sx_xlock(&sc->sc_lock);
3018         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
3019             ("DESTROY flag set on %s.", sc->sc_name));
3020         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_CLOSEWAIT) != 0,
3021             ("CLOSEWAIT flag not set on %s.", sc->sc_name));
3022         G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
3023         error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
3024         if (error != 0) {
3025                 G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
3026                     sc->sc_name, error);
3027                 sx_xunlock(&sc->sc_lock);
3028         }
3029         g_topology_lock();
3030 }
3031
3032 static int
3033 g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
3034 {
3035         struct g_mirror_softc *sc;
3036         int error = 0;
3037
3038         g_topology_assert();
3039         G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
3040             acw, ace);
3041
3042         sc = pp->private;
3043         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
3044
3045         g_topology_unlock();
3046         sx_xlock(&sc->sc_lock);
3047         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
3048             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_CLOSEWAIT) != 0 ||
3049             LIST_EMPTY(&sc->sc_disks)) {
3050                 if (acr > 0 || acw > 0 || ace > 0)
3051                         error = ENXIO;
3052                 goto end;
3053         }
3054         sc->sc_provider_open += acr + acw + ace;
3055         if (pp->acw + acw == 0)
3056                 g_mirror_idle(sc, 0);
3057         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_CLOSEWAIT) != 0 &&
3058             sc->sc_provider_open == 0)
3059                 g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK, sc, NULL);
3060 end:
3061         sx_xunlock(&sc->sc_lock);
3062         g_topology_lock();
3063         return (error);
3064 }
3065
3066 static void
3067 g_mirror_reinit_from_metadata(struct g_mirror_softc *sc,
3068     const struct g_mirror_metadata *md)
3069 {
3070
3071         sc->sc_genid = md->md_genid;
3072         sc->sc_syncid = md->md_syncid;
3073
3074         sc->sc_slice = md->md_slice;
3075         sc->sc_balance = md->md_balance;
3076         sc->sc_mediasize = md->md_mediasize;
3077         sc->sc_ndisks = md->md_all;
3078         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_MASK;
3079         sc->sc_flags |= (md->md_mflags & G_MIRROR_DEVICE_FLAG_MASK);
3080 }
3081
3082 struct g_geom *
3083 g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md,
3084     u_int type)
3085 {
3086         struct g_mirror_softc *sc;
3087         struct g_geom *gp;
3088         int error, timeout;
3089
3090         g_topology_assert();
3091         G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
3092             md->md_mid);
3093
3094         /* One disk is minimum. */
3095         if (md->md_all < 1)
3096                 return (NULL);
3097         /*
3098          * Action geom.
3099          */
3100         gp = g_new_geomf(mp, "%s", md->md_name);
3101         sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
3102         gp->start = g_mirror_start;
3103         gp->orphan = g_mirror_orphan;
3104         gp->access = g_mirror_access;
3105         gp->dumpconf = g_mirror_dumpconf;
3106
3107         sc->sc_type = type;
3108         sc->sc_id = md->md_mid;
3109         g_mirror_reinit_from_metadata(sc, md);
3110         sc->sc_sectorsize = md->md_sectorsize;
3111         sc->sc_bump_id = 0;
3112         sc->sc_idle = 1;
3113         sc->sc_last_write = time_uptime;
3114         sc->sc_writes = 0;
3115         sc->sc_refcnt = 1;
3116         sx_init(&sc->sc_lock, "gmirror:lock");
3117         TAILQ_INIT(&sc->sc_queue);
3118         mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
3119         TAILQ_INIT(&sc->sc_regular_delayed);
3120         TAILQ_INIT(&sc->sc_inflight);
3121         TAILQ_INIT(&sc->sc_sync_delayed);
3122         LIST_INIT(&sc->sc_disks);
3123         TAILQ_INIT(&sc->sc_events);
3124         mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
3125         callout_init(&sc->sc_callout, 1);
3126         mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
3127         sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
3128         gp->softc = sc;
3129         sc->sc_geom = gp;
3130         sc->sc_provider = NULL;
3131         sc->sc_provider_open = 0;
3132         /*
3133          * Synchronization geom.
3134          */
3135         gp = g_new_geomf(mp, "%s.sync", md->md_name);
3136         gp->softc = sc;
3137         gp->orphan = g_mirror_orphan;
3138         sc->sc_sync.ds_geom = gp;
3139         sc->sc_sync.ds_ndisks = 0;
3140         error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
3141             "g_mirror %s", md->md_name);
3142         if (error != 0) {
3143                 G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
3144                     sc->sc_name);
3145                 g_destroy_geom(sc->sc_sync.ds_geom);
3146                 g_destroy_geom(sc->sc_geom);
3147                 g_mirror_free_device(sc);
3148                 return (NULL);
3149         }
3150
3151         G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
3152             sc->sc_name, sc->sc_ndisks, sc->sc_id);
3153
3154         sc->sc_rootmount = root_mount_hold("GMIRROR");
3155         G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
3156         /*
3157          * Run timeout.
3158          */
3159         timeout = g_mirror_timeout * hz;
3160         callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
3161         return (sc->sc_geom);
3162 }
3163
3164 int
3165 g_mirror_destroy(struct g_mirror_softc *sc, int how)
3166 {
3167         struct g_mirror_disk *disk;
3168
3169         g_topology_assert_not();
3170         sx_assert(&sc->sc_lock, SX_XLOCKED);
3171
3172         if (sc->sc_provider_open != 0) {
3173                 switch (how) {
3174                 case G_MIRROR_DESTROY_SOFT:
3175                         G_MIRROR_DEBUG(1,
3176                             "Device %s is still open (%d).", sc->sc_name,
3177                             sc->sc_provider_open);
3178                         return (EBUSY);
3179                 case G_MIRROR_DESTROY_DELAYED:
3180                         G_MIRROR_DEBUG(1,
3181                             "Device %s will be destroyed on last close.",
3182                             sc->sc_name);
3183                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
3184                                 if (disk->d_state ==
3185                                     G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3186                                         g_mirror_sync_stop(disk, 1);
3187                                 }
3188                         }
3189                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_CLOSEWAIT;
3190                         return (EBUSY);
3191                 case G_MIRROR_DESTROY_HARD:
3192                         G_MIRROR_DEBUG(1, "Device %s is still open, so it "
3193                             "can't be definitely removed.", sc->sc_name);
3194                 }
3195         }
3196
3197         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3198                 sx_xunlock(&sc->sc_lock);
3199                 return (0);
3200         }
3201         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3202         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DRAIN;
3203         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3204         sx_xunlock(&sc->sc_lock);
3205         mtx_lock(&sc->sc_queue_mtx);
3206         wakeup(sc);
3207         mtx_unlock(&sc->sc_queue_mtx);
3208         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3209         while (sc->sc_worker != NULL)
3210                 tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3211         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3212         sx_xlock(&sc->sc_lock);
3213         g_mirror_destroy_device(sc);
3214         return (0);
3215 }
3216
3217 static void
3218 g_mirror_taste_orphan(struct g_consumer *cp)
3219 {
3220
3221         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3222             cp->provider->name));
3223 }
3224
3225 static struct g_geom *
3226 g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3227 {
3228         struct g_mirror_metadata md;
3229         struct g_mirror_softc *sc;
3230         struct g_consumer *cp;
3231         struct g_geom *gp;
3232         int error;
3233
3234         g_topology_assert();
3235         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3236         G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3237
3238         gp = g_new_geomf(mp, "mirror:taste");
3239         /*
3240          * This orphan function should be never called.
3241          */
3242         gp->orphan = g_mirror_taste_orphan;
3243         cp = g_new_consumer(gp);
3244         error = g_attach(cp, pp);
3245         if (error == 0) {
3246                 error = g_mirror_read_metadata(cp, &md);
3247                 g_detach(cp);
3248         }
3249         g_destroy_consumer(cp);
3250         g_destroy_geom(gp);
3251         if (error != 0)
3252                 return (NULL);
3253         gp = NULL;
3254
3255         if (md.md_provider[0] != '\0' &&
3256             !g_compare_names(md.md_provider, pp->name))
3257                 return (NULL);
3258         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3259                 return (NULL);
3260         if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3261                 G_MIRROR_DEBUG(0,
3262                     "Device %s: provider %s marked as inactive, skipping.",
3263                     md.md_name, pp->name);
3264                 return (NULL);
3265         }
3266         if (g_mirror_debug >= 2)
3267                 mirror_metadata_dump(&md);
3268
3269         /*
3270          * Let's check if device already exists.
3271          */
3272         sc = NULL;
3273         LIST_FOREACH(gp, &mp->geom, geom) {
3274                 sc = gp->softc;
3275                 if (sc == NULL)
3276                         continue;
3277                 if (sc->sc_type != G_MIRROR_TYPE_AUTOMATIC)
3278                         continue;
3279                 if (sc->sc_sync.ds_geom == gp)
3280                         continue;
3281                 if (strcmp(md.md_name, sc->sc_name) != 0)
3282                         continue;
3283                 if (md.md_mid != sc->sc_id) {
3284                         G_MIRROR_DEBUG(0, "Device %s already configured.",
3285                             sc->sc_name);
3286                         return (NULL);
3287                 }
3288                 break;
3289         }
3290         if (gp == NULL) {
3291                 gp = g_mirror_create(mp, &md, G_MIRROR_TYPE_AUTOMATIC);
3292                 if (gp == NULL) {
3293                         G_MIRROR_DEBUG(0, "Cannot create device %s.",
3294                             md.md_name);
3295                         return (NULL);
3296                 }
3297                 sc = gp->softc;
3298         }
3299         G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3300         g_topology_unlock();
3301         sx_xlock(&sc->sc_lock);
3302         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3303         error = g_mirror_add_disk(sc, pp, &md);
3304         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3305         if (error != 0) {
3306                 G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3307                     pp->name, gp->name, error);
3308                 if (LIST_EMPTY(&sc->sc_disks)) {
3309                         g_cancel_event(sc);
3310                         g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3311                         g_topology_lock();
3312                         return (NULL);
3313                 }
3314                 gp = NULL;
3315         }
3316         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3317                 g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3318                 g_topology_lock();
3319                 return (NULL);
3320         }
3321         sx_xunlock(&sc->sc_lock);
3322         g_topology_lock();
3323         return (gp);
3324 }
3325
3326 static void
3327 g_mirror_resize(struct g_consumer *cp)
3328 {
3329         struct g_mirror_disk *disk;
3330
3331         g_topology_assert();
3332         g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3333
3334         disk = cp->private;
3335         if (disk == NULL)
3336                 return;
3337         g_topology_unlock();
3338         g_mirror_update_metadata(disk);
3339         g_topology_lock();
3340 }
3341
3342 static int
3343 g_mirror_destroy_geom(struct gctl_req *req __unused,
3344     struct g_class *mp __unused, struct g_geom *gp)
3345 {
3346         struct g_mirror_softc *sc;
3347         int error;
3348
3349         g_topology_unlock();
3350         sc = gp->softc;
3351         sx_xlock(&sc->sc_lock);
3352         g_cancel_event(sc);
3353         error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3354         if (error != 0)
3355                 sx_xunlock(&sc->sc_lock);
3356         g_topology_lock();
3357         return (error);
3358 }
3359
3360 static void
3361 g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3362     struct g_consumer *cp, struct g_provider *pp)
3363 {
3364         struct g_mirror_softc *sc;
3365
3366         g_topology_assert();
3367
3368         sc = gp->softc;
3369         if (sc == NULL)
3370                 return;
3371         /* Skip synchronization geom. */
3372         if (gp == sc->sc_sync.ds_geom)
3373                 return;
3374         if (pp != NULL) {
3375                 /* Nothing here. */
3376         } else if (cp != NULL) {
3377                 struct g_mirror_disk *disk;
3378
3379                 disk = cp->private;
3380                 if (disk == NULL)
3381                         return;
3382                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3383                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3384                         sbuf_printf(sb, "%s<Synchronized>", indent);
3385                         if (disk->d_sync.ds_offset == 0)
3386                                 sbuf_cat(sb, "0%");
3387                         else
3388                                 sbuf_printf(sb, "%u%%",
3389                                     (u_int)((disk->d_sync.ds_offset * 100) /
3390                                     sc->sc_mediasize));
3391                         sbuf_cat(sb, "</Synchronized>\n");
3392                         if (disk->d_sync.ds_offset > 0)
3393                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3394                                     "</BytesSynced>\n", indent,
3395                                     (intmax_t)disk->d_sync.ds_offset);
3396                 }
3397                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3398                     disk->d_sync.ds_syncid);
3399                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3400                     disk->d_genid);
3401                 sbuf_printf(sb, "%s<Flags>", indent);
3402                 if (disk->d_flags == 0)
3403                         sbuf_cat(sb, "NONE");
3404                 else {
3405                         int first = 1;
3406
3407 #define ADD_FLAG(flag, name)    do {                                    \
3408         if ((disk->d_flags & (flag)) != 0) {                            \
3409                 if (!first)                                             \
3410                         sbuf_cat(sb, ", ");                             \
3411                 else                                                    \
3412                         first = 0;                                      \
3413                 sbuf_cat(sb, name);                                     \
3414         }                                                               \
3415 } while (0)
3416                         ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3417                         ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3418                         ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3419                         ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3420                             "SYNCHRONIZING");
3421                         ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3422                         ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3423 #undef  ADD_FLAG
3424                 }
3425                 sbuf_cat(sb, "</Flags>\n");
3426                 sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3427                     disk->d_priority);
3428                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3429                     g_mirror_disk_state2str(disk->d_state));
3430         } else {
3431                 sbuf_printf(sb, "%s<Type>", indent);
3432                 switch (sc->sc_type) {
3433                 case G_MIRROR_TYPE_AUTOMATIC:
3434                         sbuf_cat(sb, "AUTOMATIC");
3435                         break;
3436                 case G_MIRROR_TYPE_MANUAL:
3437                         sbuf_cat(sb, "MANUAL");
3438                         break;
3439                 default:
3440                         sbuf_cat(sb, "UNKNOWN");
3441                         break;
3442                 }
3443                 sbuf_cat(sb, "</Type>\n");
3444                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3445                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3446                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3447                 sbuf_printf(sb, "%s<Flags>", indent);
3448                 if (sc->sc_flags == 0)
3449                         sbuf_cat(sb, "NONE");
3450                 else {
3451                         int first = 1;
3452
3453 #define ADD_FLAG(flag, name)    do {                                    \
3454         if ((sc->sc_flags & (flag)) != 0) {                             \
3455                 if (!first)                                             \
3456                         sbuf_cat(sb, ", ");                             \
3457                 else                                                    \
3458                         first = 0;                                      \
3459                 sbuf_cat(sb, name);                                     \
3460         }                                                               \
3461 } while (0)
3462                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3463                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3464 #undef  ADD_FLAG
3465                 }
3466                 sbuf_cat(sb, "</Flags>\n");
3467                 sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3468                     (u_int)sc->sc_slice);
3469                 sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3470                     balance_name(sc->sc_balance));
3471                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3472                     sc->sc_ndisks);
3473                 sbuf_printf(sb, "%s<State>", indent);
3474                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3475                         sbuf_printf(sb, "%s", "STARTING");
3476                 else if (sc->sc_ndisks ==
3477                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3478                         sbuf_printf(sb, "%s", "COMPLETE");
3479                 else
3480                         sbuf_printf(sb, "%s", "DEGRADED");
3481                 sbuf_cat(sb, "</State>\n");
3482         }
3483 }
3484
3485 static void
3486 g_mirror_shutdown_post_sync(void *arg, int howto)
3487 {
3488         struct g_class *mp;
3489         struct g_geom *gp, *gp2;
3490         struct g_mirror_softc *sc;
3491         int error;
3492
3493         if (KERNEL_PANICKED())
3494                 return;
3495
3496         mp = arg;
3497         g_topology_lock();
3498         g_mirror_shutdown = 1;
3499         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3500                 if ((sc = gp->softc) == NULL)
3501                         continue;
3502                 /* Skip synchronization geom. */
3503                 if (gp == sc->sc_sync.ds_geom)
3504                         continue;
3505                 g_topology_unlock();
3506                 sx_xlock(&sc->sc_lock);
3507                 g_mirror_idle(sc, -1);
3508                 g_cancel_event(sc);
3509                 error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3510                 if (error != 0)
3511                         sx_xunlock(&sc->sc_lock);
3512                 g_topology_lock();
3513         }
3514         g_topology_unlock();
3515 }
3516
3517 static void
3518 g_mirror_init(struct g_class *mp)
3519 {
3520
3521         g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3522             g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3523         if (g_mirror_post_sync == NULL)
3524                 G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3525 }
3526
3527 static void
3528 g_mirror_fini(struct g_class *mp)
3529 {
3530
3531         if (g_mirror_post_sync != NULL)
3532                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3533 }
3534
3535 /*
3536  * Refresh the mirror device's metadata when gmirror encounters a newer
3537  * generation as the individual components are being added to the mirror set.
3538  */
3539 static int
3540 g_mirror_refresh_device(struct g_mirror_softc *sc, const struct g_provider *pp,
3541     const struct g_mirror_metadata *md)
3542 {
3543
3544         g_topology_assert_not();
3545         sx_assert(&sc->sc_lock, SX_XLOCKED);
3546
3547         KASSERT(sc->sc_genid <= md->md_genid,
3548             ("%s: attempted to refresh from stale component %s (device %s) "
3549             "(%u < %u).", __func__, pp->name, sc->sc_name, md->md_genid,
3550             sc->sc_genid));
3551
3552         if (sc->sc_genid > md->md_genid || (sc->sc_genid == md->md_genid &&
3553             sc->sc_syncid >= md->md_syncid))
3554                 return (0);
3555
3556         G_MIRROR_DEBUG(0, "Found newer version for device %s (genid: curr=%u "
3557             "new=%u; syncid: curr=%u new=%u; ndisks: curr=%u new=%u; "
3558             "provider=%s).", sc->sc_name, sc->sc_genid, md->md_genid,
3559             sc->sc_syncid, md->md_syncid, sc->sc_ndisks, md->md_all, pp->name);
3560
3561         if (sc->sc_state != G_MIRROR_DEVICE_STATE_STARTING) {
3562                 /* Probable data corruption detected */
3563                 G_MIRROR_DEBUG(0, "Cannot refresh metadata in %s state "
3564                     "(device=%s genid=%u). A stale mirror device was launched.",
3565                     g_mirror_device_state2str(sc->sc_state), sc->sc_name,
3566                     sc->sc_genid);
3567                 return (EINVAL);
3568         }
3569
3570         /* Update softc */
3571         g_mirror_reinit_from_metadata(sc, md);
3572
3573         G_MIRROR_DEBUG(1, "Refresh device %s (id=%u, state=%s) from disk %s "
3574             "(genid=%u syncid=%u md_all=%u).", sc->sc_name, md->md_mid,
3575             g_mirror_device_state2str(sc->sc_state), pp->name, md->md_genid,
3576             md->md_syncid, (unsigned)md->md_all);
3577
3578         return (0);
3579 }
3580
3581 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);
3582 MODULE_VERSION(geom_mirror, 0);