sys/geom/mirror/g_mirror.c

   1 /*-
   2  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/module.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/bio.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/malloc.h>
  41 #include <sys/eventhandler.h>
  42 #include <vm/uma.h>
  43 #include <geom/geom.h>
  44 #include <sys/proc.h>
  45 #include <sys/kthread.h>
  46 #include <sys/sched.h>
  47 #include <geom/mirror/g_mirror.h>
  48
  49 FEATURE(geom_mirror, "GEOM mirroring support");
  50
  51 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
  52
  53 SYSCTL_DECL(_kern_geom);
  54 static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
  55     "GEOM_MIRROR stuff");
  56 u_int g_mirror_debug = 0;
  57 TUNABLE_INT("kern.geom.mirror.debug", &g_mirror_debug);
  58 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RW, &g_mirror_debug, 0,
  59     "Debug level");
  60 static u_int g_mirror_timeout = 4;
  61 TUNABLE_INT("kern.geom.mirror.timeout", &g_mirror_timeout);
  62 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RW, &g_mirror_timeout,
  63     0, "Time to wait on all mirror components");
  64 static u_int g_mirror_idletime = 5;
  65 TUNABLE_INT("kern.geom.mirror.idletime", &g_mirror_idletime);
  66 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RW,
  67     &g_mirror_idletime, 0, "Mark components as clean when idling");
  68 static u_int g_mirror_disconnect_on_failure = 1;
  69 TUNABLE_INT("kern.geom.mirror.disconnect_on_failure",
  70     &g_mirror_disconnect_on_failure);
  71 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RW,
  72     &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  73 static u_int g_mirror_syncreqs = 2;
  74 TUNABLE_INT("kern.geom.mirror.sync_requests", &g_mirror_syncreqs);
  75 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  76     &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
  77
  78 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  79         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));       \
  80         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  81         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));       \
  82 } while (0)
  83
  84 static eventhandler_tag g_mirror_post_sync = NULL;
  85 static int g_mirror_shutdown = 0;
  86
  87 static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
  88     struct g_geom *gp);
  89 static g_taste_t g_mirror_taste;
  90 static g_resize_t g_mirror_resize;
  91 static void g_mirror_init(struct g_class *mp);
  92 static void g_mirror_fini(struct g_class *mp);
  93
  94 struct g_class g_mirror_class = {
  95         .name = G_MIRROR_CLASS_NAME,
  96         .version = G_VERSION,
  97         .ctlreq = g_mirror_config,
  98         .taste = g_mirror_taste,
  99         .destroy_geom = g_mirror_destroy_geom,
 100         .init = g_mirror_init,
 101         .fini = g_mirror_fini,
 102         .resize = g_mirror_resize
 103 };
 104
 105
 106 static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
 107 static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
 108 static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
 109 static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
 110     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 111 static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
 112 static void g_mirror_register_request(struct bio *bp);
 113 static void g_mirror_sync_release(struct g_mirror_softc *sc);
 114
 115
 116 static const char *
 117 g_mirror_disk_state2str(int state)
 118 {
 119
 120         switch (state) {
 121         case G_MIRROR_DISK_STATE_NONE:
 122                 return ("NONE");
 123         case G_MIRROR_DISK_STATE_NEW:
 124                 return ("NEW");
 125         case G_MIRROR_DISK_STATE_ACTIVE:
 126                 return ("ACTIVE");
 127         case G_MIRROR_DISK_STATE_STALE:
 128                 return ("STALE");
 129         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 130                 return ("SYNCHRONIZING");
 131         case G_MIRROR_DISK_STATE_DISCONNECTED:
 132                 return ("DISCONNECTED");
 133         case G_MIRROR_DISK_STATE_DESTROY:
 134                 return ("DESTROY");
 135         default:
 136                 return ("INVALID");
 137         }
 138 }
 139
 140 static const char *
 141 g_mirror_device_state2str(int state)
 142 {
 143
 144         switch (state) {
 145         case G_MIRROR_DEVICE_STATE_STARTING:
 146                 return ("STARTING");
 147         case G_MIRROR_DEVICE_STATE_RUNNING:
 148                 return ("RUNNING");
 149         default:
 150                 return ("INVALID");
 151         }
 152 }
 153
 154 static const char *
 155 g_mirror_get_diskname(struct g_mirror_disk *disk)
 156 {
 157
 158         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 159                 return ("[unknown]");
 160         return (disk->d_name);
 161 }
 162
 163 /*
 164  * --- Events handling functions ---
 165  * Events in geom_mirror are used to maintain disks and device status
 166  * from one thread to simplify locking.
 167  */
 168 static void
 169 g_mirror_event_free(struct g_mirror_event *ep)
 170 {
 171
 172         free(ep, M_MIRROR);
 173 }
 174
 175 int
 176 g_mirror_event_send(void *arg, int state, int flags)
 177 {
 178         struct g_mirror_softc *sc;
 179         struct g_mirror_disk *disk;
 180         struct g_mirror_event *ep;
 181         int error;
 182
 183         ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
 184         G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 185         if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
 186                 disk = NULL;
 187                 sc = arg;
 188         } else {
 189                 disk = arg;
 190                 sc = disk->d_softc;
 191         }
 192         ep->e_disk = disk;
 193         ep->e_state = state;
 194         ep->e_flags = flags;
 195         ep->e_error = 0;
 196         mtx_lock(&sc->sc_events_mtx);
 197         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 198         mtx_unlock(&sc->sc_events_mtx);
 199         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 200         mtx_lock(&sc->sc_queue_mtx);
 201         wakeup(sc);
 202         mtx_unlock(&sc->sc_queue_mtx);
 203         if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 204                 return (0);
 205         sx_assert(&sc->sc_lock, SX_XLOCKED);
 206         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 207         sx_xunlock(&sc->sc_lock);
 208         while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
 209                 mtx_lock(&sc->sc_events_mtx);
 210                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
 211                     hz * 5);
 212         }
 213         error = ep->e_error;
 214         g_mirror_event_free(ep);
 215         sx_xlock(&sc->sc_lock);
 216         return (error);
 217 }
 218
 219 static struct g_mirror_event *
 220 g_mirror_event_get(struct g_mirror_softc *sc)
 221 {
 222         struct g_mirror_event *ep;
 223
 224         mtx_lock(&sc->sc_events_mtx);
 225         ep = TAILQ_FIRST(&sc->sc_events);
 226         mtx_unlock(&sc->sc_events_mtx);
 227         return (ep);
 228 }
 229
 230 static void
 231 g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
 232 {
 233
 234         mtx_lock(&sc->sc_events_mtx);
 235         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 236         mtx_unlock(&sc->sc_events_mtx);
 237 }
 238
 239 static void
 240 g_mirror_event_cancel(struct g_mirror_disk *disk)
 241 {
 242         struct g_mirror_softc *sc;
 243         struct g_mirror_event *ep, *tmpep;
 244
 245         sc = disk->d_softc;
 246         sx_assert(&sc->sc_lock, SX_XLOCKED);
 247
 248         mtx_lock(&sc->sc_events_mtx);
 249         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 250                 if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
 251                         continue;
 252                 if (ep->e_disk != disk)
 253                         continue;
 254                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 255                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 256                         g_mirror_event_free(ep);
 257                 else {
 258                         ep->e_error = ECANCELED;
 259                         wakeup(ep);
 260                 }
 261         }
 262         mtx_unlock(&sc->sc_events_mtx);
 263 }
 264
 265 /*
 266  * Return the number of disks in given state.
 267  * If state is equal to -1, count all connected disks.
 268  */
 269 u_int
 270 g_mirror_ndisks(struct g_mirror_softc *sc, int state)
 271 {
 272         struct g_mirror_disk *disk;
 273         u_int n = 0;
 274
 275         sx_assert(&sc->sc_lock, SX_LOCKED);
 276
 277         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 278                 if (state == -1 || disk->d_state == state)
 279                         n++;
 280         }
 281         return (n);
 282 }
 283
 284 /*
 285  * Find a disk in mirror by its disk ID.
 286  */
 287 static struct g_mirror_disk *
 288 g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
 289 {
 290         struct g_mirror_disk *disk;
 291
 292         sx_assert(&sc->sc_lock, SX_XLOCKED);
 293
 294         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 295                 if (disk->d_id == id)
 296                         return (disk);
 297         }
 298         return (NULL);
 299 }
 300
 301 static u_int
 302 g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
 303 {
 304         struct bio *bp;
 305         u_int nreqs = 0;
 306
 307         mtx_lock(&sc->sc_queue_mtx);
 308         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 309                 if (bp->bio_from == cp)
 310                         nreqs++;
 311         }
 312         mtx_unlock(&sc->sc_queue_mtx);
 313         return (nreqs);
 314 }
 315
 316 static int
 317 g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
 318 {
 319
 320         if (cp->index > 0) {
 321                 G_MIRROR_DEBUG(2,
 322                     "I/O requests for %s exist, can't destroy it now.",
 323                     cp->provider->name);
 324                 return (1);
 325         }
 326         if (g_mirror_nrequests(sc, cp) > 0) {
 327                 G_MIRROR_DEBUG(2,
 328                     "I/O requests for %s in queue, can't destroy it now.",
 329                     cp->provider->name);
 330                 return (1);
 331         }
 332         return (0);
 333 }
 334
 335 static void
 336 g_mirror_destroy_consumer(void *arg, int flags __unused)
 337 {
 338         struct g_consumer *cp;
 339
 340         g_topology_assert();
 341
 342         cp = arg;
 343         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 344         g_detach(cp);
 345         g_destroy_consumer(cp);
 346 }
 347
 348 static void
 349 g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 350 {
 351         struct g_provider *pp;
 352         int retaste_wait;
 353
 354         g_topology_assert();
 355
 356         cp->private = NULL;
 357         if (g_mirror_is_busy(sc, cp))
 358                 return;
 359         pp = cp->provider;
 360         retaste_wait = 0;
 361         if (cp->acw == 1) {
 362                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 363                         retaste_wait = 1;
 364         }
 365         G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 366             -cp->acw, -cp->ace, 0);
 367         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 368                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 369         if (retaste_wait) {
 370                 /*
 371                  * After retaste event was send (inside g_access()), we can send
 372                  * event to detach and destroy consumer.
 373                  * A class, which has consumer to the given provider connected
 374                  * will not receive retaste event for the provider.
 375                  * This is the way how I ignore retaste events when I close
 376                  * consumers opened for write: I detach and destroy consumer
 377                  * after retaste event is sent.
 378                  */
 379                 g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
 380                 return;
 381         }
 382         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
 383         g_detach(cp);
 384         g_destroy_consumer(cp);
 385 }
 386
 387 static int
 388 g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
 389 {
 390         struct g_consumer *cp;
 391         int error;
 392
 393         g_topology_assert_not();
 394         KASSERT(disk->d_consumer == NULL,
 395             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 396
 397         g_topology_lock();
 398         cp = g_new_consumer(disk->d_softc->sc_geom);
 399         cp->flags |= G_CF_DIRECT_RECEIVE;
 400         error = g_attach(cp, pp);
 401         if (error != 0) {
 402                 g_destroy_consumer(cp);
 403                 g_topology_unlock();
 404                 return (error);
 405         }
 406         error = g_access(cp, 1, 1, 1);
 407         if (error != 0) {
 408                 g_detach(cp);
 409                 g_destroy_consumer(cp);
 410                 g_topology_unlock();
 411                 G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
 412                     pp->name, error);
 413                 return (error);
 414         }
 415         g_topology_unlock();
 416         disk->d_consumer = cp;
 417         disk->d_consumer->private = disk;
 418         disk->d_consumer->index = 0;
 419
 420         G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
 421         return (0);
 422 }
 423
 424 static void
 425 g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 426 {
 427
 428         g_topology_assert();
 429
 430         if (cp == NULL)
 431                 return;
 432         if (cp->provider != NULL)
 433                 g_mirror_kill_consumer(sc, cp);
 434         else
 435                 g_destroy_consumer(cp);
 436 }
 437
 438 /*
 439  * Initialize disk. This means allocate memory, create consumer, attach it
 440  * to the provider and open access (r1w1e1) to it.
 441  */
 442 static struct g_mirror_disk *
 443 g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 444     struct g_mirror_metadata *md, int *errorp)
 445 {
 446         struct g_mirror_disk *disk;
 447         int i, error;
 448
 449         disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
 450         if (disk == NULL) {
 451                 error = ENOMEM;
 452                 goto fail;
 453         }
 454         disk->d_softc = sc;
 455         error = g_mirror_connect_disk(disk, pp);
 456         if (error != 0)
 457                 goto fail;
 458         disk->d_id = md->md_did;
 459         disk->d_state = G_MIRROR_DISK_STATE_NONE;
 460         disk->d_priority = md->md_priority;
 461         disk->d_flags = md->md_dflags;
 462         error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
 463         if (error == 0 && i != 0)
 464                 disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 465         if (md->md_provider[0] != '\0')
 466                 disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
 467         disk->d_sync.ds_consumer = NULL;
 468         disk->d_sync.ds_offset = md->md_sync_offset;
 469         disk->d_sync.ds_offset_done = md->md_sync_offset;
 470         disk->d_genid = md->md_genid;
 471         disk->d_sync.ds_syncid = md->md_syncid;
 472         if (errorp != NULL)
 473                 *errorp = 0;
 474         return (disk);
 475 fail:
 476         if (errorp != NULL)
 477                 *errorp = error;
 478         if (disk != NULL)
 479                 free(disk, M_MIRROR);
 480         return (NULL);
 481 }
 482
 483 static void
 484 g_mirror_destroy_disk(struct g_mirror_disk *disk)
 485 {
 486         struct g_mirror_softc *sc;
 487
 488         g_topology_assert_not();
 489         sc = disk->d_softc;
 490         sx_assert(&sc->sc_lock, SX_XLOCKED);
 491
 492         LIST_REMOVE(disk, d_next);
 493         g_mirror_event_cancel(disk);
 494         if (sc->sc_hint == disk)
 495                 sc->sc_hint = NULL;
 496         switch (disk->d_state) {
 497         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 498                 g_mirror_sync_stop(disk, 1);
 499                 /* FALLTHROUGH */
 500         case G_MIRROR_DISK_STATE_NEW:
 501         case G_MIRROR_DISK_STATE_STALE:
 502         case G_MIRROR_DISK_STATE_ACTIVE:
 503                 g_topology_lock();
 504                 g_mirror_disconnect_consumer(sc, disk->d_consumer);
 505                 g_topology_unlock();
 506                 free(disk, M_MIRROR);
 507                 break;
 508         default:
 509                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 510                     g_mirror_get_diskname(disk),
 511                     g_mirror_disk_state2str(disk->d_state)));
 512         }
 513 }
 514
 515 static void
 516 g_mirror_destroy_device(struct g_mirror_softc *sc)
 517 {
 518         struct g_mirror_disk *disk;
 519         struct g_mirror_event *ep;
 520         struct g_geom *gp;
 521         struct g_consumer *cp, *tmpcp;
 522
 523         g_topology_assert_not();
 524         sx_assert(&sc->sc_lock, SX_XLOCKED);
 525
 526         gp = sc->sc_geom;
 527         if (sc->sc_provider != NULL)
 528                 g_mirror_destroy_provider(sc);
 529         for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
 530             disk = LIST_FIRST(&sc->sc_disks)) {
 531                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 532                 g_mirror_update_metadata(disk);
 533                 g_mirror_destroy_disk(disk);
 534         }
 535         while ((ep = g_mirror_event_get(sc)) != NULL) {
 536                 g_mirror_event_remove(sc, ep);
 537                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 538                         g_mirror_event_free(ep);
 539                 else {
 540                         ep->e_error = ECANCELED;
 541                         ep->e_flags |= G_MIRROR_EVENT_DONE;
 542                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 543                         mtx_lock(&sc->sc_events_mtx);
 544                         wakeup(ep);
 545                         mtx_unlock(&sc->sc_events_mtx);
 546                 }
 547         }
 548         callout_drain(&sc->sc_callout);
 549
 550         g_topology_lock();
 551         LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
 552                 g_mirror_disconnect_consumer(sc, cp);
 553         }
 554         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 555         G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
 556         g_wither_geom(gp, ENXIO);
 557         g_topology_unlock();
 558         mtx_destroy(&sc->sc_queue_mtx);
 559         mtx_destroy(&sc->sc_events_mtx);
 560         mtx_destroy(&sc->sc_done_mtx);
 561         sx_xunlock(&sc->sc_lock);
 562         sx_destroy(&sc->sc_lock);
 563 }
 564
 565 static void
 566 g_mirror_orphan(struct g_consumer *cp)
 567 {
 568         struct g_mirror_disk *disk;
 569
 570         g_topology_assert();
 571
 572         disk = cp->private;
 573         if (disk == NULL)
 574                 return;
 575         disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 576         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 577             G_MIRROR_EVENT_DONTWAIT);
 578 }
 579
 580 /*
 581  * Function should return the next active disk on the list.
 582  * It is possible that it will be the same disk as given.
 583  * If there are no active disks on list, NULL is returned.
 584  */
 585 static __inline struct g_mirror_disk *
 586 g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
 587 {
 588         struct g_mirror_disk *dp;
 589
 590         for (dp = LIST_NEXT(disk, d_next); dp != disk;
 591             dp = LIST_NEXT(dp, d_next)) {
 592                 if (dp == NULL)
 593                         dp = LIST_FIRST(&sc->sc_disks);
 594                 if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
 595                         break;
 596         }
 597         if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 598                 return (NULL);
 599         return (dp);
 600 }
 601
 602 static struct g_mirror_disk *
 603 g_mirror_get_disk(struct g_mirror_softc *sc)
 604 {
 605         struct g_mirror_disk *disk;
 606
 607         if (sc->sc_hint == NULL) {
 608                 sc->sc_hint = LIST_FIRST(&sc->sc_disks);
 609                 if (sc->sc_hint == NULL)
 610                         return (NULL);
 611         }
 612         disk = sc->sc_hint;
 613         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
 614                 disk = g_mirror_find_next(sc, disk);
 615                 if (disk == NULL)
 616                         return (NULL);
 617         }
 618         sc->sc_hint = g_mirror_find_next(sc, disk);
 619         return (disk);
 620 }
 621
 622 static int
 623 g_mirror_write_metadata(struct g_mirror_disk *disk,
 624     struct g_mirror_metadata *md)
 625 {
 626         struct g_mirror_softc *sc;
 627         struct g_consumer *cp;
 628         off_t offset, length;
 629         u_char *sector;
 630         int error = 0;
 631
 632         g_topology_assert_not();
 633         sc = disk->d_softc;
 634         sx_assert(&sc->sc_lock, SX_LOCKED);
 635
 636         cp = disk->d_consumer;
 637         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 638         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 639         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 640             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 641             cp->acw, cp->ace));
 642         length = cp->provider->sectorsize;
 643         offset = cp->provider->mediasize - length;
 644         sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
 645         if (md != NULL &&
 646             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
 647                 /*
 648                  * Handle the case, when the size of parent provider reduced.
 649                  */
 650                 if (offset < md->md_mediasize)
 651                         error = ENOSPC;
 652                 else
 653                         mirror_metadata_encode(md, sector);
 654         }
 655         if (error == 0)
 656                 error = g_write_data(cp, offset, sector, length);
 657         free(sector, M_MIRROR);
 658         if (error != 0) {
 659                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 660                         disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 661                         G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
 662                             "(device=%s, error=%d).",
 663                             g_mirror_get_diskname(disk), sc->sc_name, error);
 664                 } else {
 665                         G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
 666                             "(device=%s, error=%d).",
 667                             g_mirror_get_diskname(disk), sc->sc_name, error);
 668                 }
 669                 if (g_mirror_disconnect_on_failure &&
 670                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 671                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 672                         g_mirror_event_send(disk,
 673                             G_MIRROR_DISK_STATE_DISCONNECTED,
 674                             G_MIRROR_EVENT_DONTWAIT);
 675                 }
 676         }
 677         return (error);
 678 }
 679
 680 static int
 681 g_mirror_clear_metadata(struct g_mirror_disk *disk)
 682 {
 683         int error;
 684
 685         g_topology_assert_not();
 686         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 687
 688         error = g_mirror_write_metadata(disk, NULL);
 689         if (error == 0) {
 690                 G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
 691                     g_mirror_get_diskname(disk));
 692         } else {
 693                 G_MIRROR_DEBUG(0,
 694                     "Cannot clear metadata on disk %s (error=%d).",
 695                     g_mirror_get_diskname(disk), error);
 696         }
 697         return (error);
 698 }
 699
 700 void
 701 g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
 702     struct g_mirror_metadata *md)
 703 {
 704
 705         strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
 706         md->md_version = G_MIRROR_VERSION;
 707         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 708         md->md_mid = sc->sc_id;
 709         md->md_all = sc->sc_ndisks;
 710         md->md_slice = sc->sc_slice;
 711         md->md_balance = sc->sc_balance;
 712         md->md_genid = sc->sc_genid;
 713         md->md_mediasize = sc->sc_mediasize;
 714         md->md_sectorsize = sc->sc_sectorsize;
 715         md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
 716         bzero(md->md_provider, sizeof(md->md_provider));
 717         if (disk == NULL) {
 718                 md->md_did = arc4random();
 719                 md->md_priority = 0;
 720                 md->md_syncid = 0;
 721                 md->md_dflags = 0;
 722                 md->md_sync_offset = 0;
 723                 md->md_provsize = 0;
 724         } else {
 725                 md->md_did = disk->d_id;
 726                 md->md_priority = disk->d_priority;
 727                 md->md_syncid = disk->d_sync.ds_syncid;
 728                 md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
 729                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
 730                         md->md_sync_offset = disk->d_sync.ds_offset_done;
 731                 else
 732                         md->md_sync_offset = 0;
 733                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
 734                         strlcpy(md->md_provider,
 735                             disk->d_consumer->provider->name,
 736                             sizeof(md->md_provider));
 737                 }
 738                 md->md_provsize = disk->d_consumer->provider->mediasize;
 739         }
 740 }
 741
 742 void
 743 g_mirror_update_metadata(struct g_mirror_disk *disk)
 744 {
 745         struct g_mirror_softc *sc;
 746         struct g_mirror_metadata md;
 747         int error;
 748
 749         g_topology_assert_not();
 750         sc = disk->d_softc;
 751         sx_assert(&sc->sc_lock, SX_LOCKED);
 752
 753         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
 754                 g_mirror_fill_metadata(sc, disk, &md);
 755         error = g_mirror_write_metadata(disk, &md);
 756         if (error == 0) {
 757                 G_MIRROR_DEBUG(2, "Metadata on %s updated.",
 758                     g_mirror_get_diskname(disk));
 759         } else {
 760                 G_MIRROR_DEBUG(0,
 761                     "Cannot update metadata on disk %s (error=%d).",
 762                     g_mirror_get_diskname(disk), error);
 763         }
 764 }
 765
 766 static void
 767 g_mirror_bump_syncid(struct g_mirror_softc *sc)
 768 {
 769         struct g_mirror_disk *disk;
 770
 771         g_topology_assert_not();
 772         sx_assert(&sc->sc_lock, SX_XLOCKED);
 773         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 774             ("%s called with no active disks (device=%s).", __func__,
 775             sc->sc_name));
 776
 777         sc->sc_syncid++;
 778         G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 779             sc->sc_syncid);
 780         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 781                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 782                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 783                         disk->d_sync.ds_syncid = sc->sc_syncid;
 784                         g_mirror_update_metadata(disk);
 785                 }
 786         }
 787 }
 788
 789 static void
 790 g_mirror_bump_genid(struct g_mirror_softc *sc)
 791 {
 792         struct g_mirror_disk *disk;
 793
 794         g_topology_assert_not();
 795         sx_assert(&sc->sc_lock, SX_XLOCKED);
 796         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 797             ("%s called with no active disks (device=%s).", __func__,
 798             sc->sc_name));
 799
 800         sc->sc_genid++;
 801         G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 802             sc->sc_genid);
 803         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 804                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 805                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 806                         disk->d_genid = sc->sc_genid;
 807                         g_mirror_update_metadata(disk);
 808                 }
 809         }
 810 }
 811
 812 static int
 813 g_mirror_idle(struct g_mirror_softc *sc, int acw)
 814 {
 815         struct g_mirror_disk *disk;
 816         int timeout;
 817
 818         g_topology_assert_not();
 819         sx_assert(&sc->sc_lock, SX_XLOCKED);
 820
 821         if (sc->sc_provider == NULL)
 822                 return (0);
 823         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 824                 return (0);
 825         if (sc->sc_idle)
 826                 return (0);
 827         if (sc->sc_writes > 0)
 828                 return (0);
 829         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 830                 timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
 831                 if (!g_mirror_shutdown && timeout > 0)
 832                         return (timeout);
 833         }
 834         sc->sc_idle = 1;
 835         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 836                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 837                         continue;
 838                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
 839                     g_mirror_get_diskname(disk), sc->sc_name);
 840                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 841                 g_mirror_update_metadata(disk);
 842         }
 843         return (0);
 844 }
 845
 846 static void
 847 g_mirror_unidle(struct g_mirror_softc *sc)
 848 {
 849         struct g_mirror_disk *disk;
 850
 851         g_topology_assert_not();
 852         sx_assert(&sc->sc_lock, SX_XLOCKED);
 853
 854         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 855                 return;
 856         sc->sc_idle = 0;
 857         sc->sc_last_write = time_uptime;
 858         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 859                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 860                         continue;
 861                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
 862                     g_mirror_get_diskname(disk), sc->sc_name);
 863                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
 864                 g_mirror_update_metadata(disk);
 865         }
 866 }
 867
 868 static void
 869 g_mirror_flush_done(struct bio *bp)
 870 {
 871         struct g_mirror_softc *sc;
 872         struct bio *pbp;
 873
 874         pbp = bp->bio_parent;
 875         sc = pbp->bio_to->geom->softc;
 876         mtx_lock(&sc->sc_done_mtx);
 877         if (pbp->bio_error == 0)
 878                 pbp->bio_error = bp->bio_error;
 879         pbp->bio_completed += bp->bio_completed;
 880         pbp->bio_inbed++;
 881         if (pbp->bio_children == pbp->bio_inbed) {
 882                 mtx_unlock(&sc->sc_done_mtx);
 883                 g_io_deliver(pbp, pbp->bio_error);
 884         } else
 885                 mtx_unlock(&sc->sc_done_mtx);
 886         g_destroy_bio(bp);
 887 }
 888
 889 static void
 890 g_mirror_done(struct bio *bp)
 891 {
 892         struct g_mirror_softc *sc;
 893
 894         sc = bp->bio_from->geom->softc;
 895         bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
 896         mtx_lock(&sc->sc_queue_mtx);
 897         bioq_disksort(&sc->sc_queue, bp);
 898         mtx_unlock(&sc->sc_queue_mtx);
 899         wakeup(sc);
 900 }
 901
 902 static void
 903 g_mirror_regular_request(struct bio *bp)
 904 {
 905         struct g_mirror_softc *sc;
 906         struct g_mirror_disk *disk;
 907         struct bio *pbp;
 908
 909         g_topology_assert_not();
 910
 911         pbp = bp->bio_parent;
 912         sc = pbp->bio_to->geom->softc;
 913         bp->bio_from->index--;
 914         if (bp->bio_cmd == BIO_WRITE)
 915                 sc->sc_writes--;
 916         disk = bp->bio_from->private;
 917         if (disk == NULL) {
 918                 g_topology_lock();
 919                 g_mirror_kill_consumer(sc, bp->bio_from);
 920                 g_topology_unlock();
 921         }
 922
 923         pbp->bio_inbed++;
 924         KASSERT(pbp->bio_inbed <= pbp->bio_children,
 925             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
 926             pbp->bio_children));
 927         if (bp->bio_error == 0 && pbp->bio_error == 0) {
 928                 G_MIRROR_LOGREQ(3, bp, "Request delivered.");
 929                 g_destroy_bio(bp);
 930                 if (pbp->bio_children == pbp->bio_inbed) {
 931                         G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
 932                         pbp->bio_completed = pbp->bio_length;
 933                         if (pbp->bio_cmd == BIO_WRITE ||
 934                             pbp->bio_cmd == BIO_DELETE) {
 935                                 bioq_remove(&sc->sc_inflight, pbp);
 936                                 /* Release delayed sync requests if possible. */
 937                                 g_mirror_sync_release(sc);
 938                         }
 939                         g_io_deliver(pbp, pbp->bio_error);
 940                 }
 941                 return;
 942         } else if (bp->bio_error != 0) {
 943                 if (pbp->bio_error == 0)
 944                         pbp->bio_error = bp->bio_error;
 945                 if (disk != NULL) {
 946                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 947                                 disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 948                                 G_MIRROR_LOGREQ(0, bp,
 949                                     "Request failed (error=%d).",
 950                                     bp->bio_error);
 951                         } else {
 952                                 G_MIRROR_LOGREQ(1, bp,
 953                                     "Request failed (error=%d).",
 954                                     bp->bio_error);
 955                         }
 956                         if (g_mirror_disconnect_on_failure &&
 957                             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
 958                         {
 959                                 sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 960                                 g_mirror_event_send(disk,
 961                                     G_MIRROR_DISK_STATE_DISCONNECTED,
 962                                     G_MIRROR_EVENT_DONTWAIT);
 963                         }
 964                 }
 965                 switch (pbp->bio_cmd) {
 966                 case BIO_DELETE:
 967                 case BIO_WRITE:
 968                         pbp->bio_inbed--;
 969                         pbp->bio_children--;
 970                         break;
 971                 }
 972         }
 973         g_destroy_bio(bp);
 974
 975         switch (pbp->bio_cmd) {
 976         case BIO_READ:
 977                 if (pbp->bio_inbed < pbp->bio_children)
 978                         break;
 979                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
 980                         g_io_deliver(pbp, pbp->bio_error);
 981                 else {
 982                         pbp->bio_error = 0;
 983                         mtx_lock(&sc->sc_queue_mtx);
 984                         bioq_disksort(&sc->sc_queue, pbp);
 985                         mtx_unlock(&sc->sc_queue_mtx);
 986                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 987                         wakeup(sc);
 988                 }
 989                 break;
 990         case BIO_DELETE:
 991         case BIO_WRITE:
 992                 if (pbp->bio_children == 0) {
 993                         /*
 994                          * All requests failed.
 995                          */
 996                 } else if (pbp->bio_inbed < pbp->bio_children) {
 997                         /* Do nothing. */
 998                         break;
 999                 } else if (pbp->bio_children == pbp->bio_inbed) {
1000                         /* Some requests succeeded. */
1001                         pbp->bio_error = 0;
1002                         pbp->bio_completed = pbp->bio_length;
1003                 }
1004                 bioq_remove(&sc->sc_inflight, pbp);
1005                 /* Release delayed sync requests if possible. */
1006                 g_mirror_sync_release(sc);
1007                 g_io_deliver(pbp, pbp->bio_error);
1008                 break;
1009         default:
1010                 KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1011                 break;
1012         }
1013 }
1014
1015 static void
1016 g_mirror_sync_done(struct bio *bp)
1017 {
1018         struct g_mirror_softc *sc;
1019
1020         G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1021         sc = bp->bio_from->geom->softc;
1022         bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1023         mtx_lock(&sc->sc_queue_mtx);
1024         bioq_disksort(&sc->sc_queue, bp);
1025         mtx_unlock(&sc->sc_queue_mtx);
1026         wakeup(sc);
1027 }
1028
1029 static void
1030 g_mirror_kernel_dump(struct bio *bp)
1031 {
1032         struct g_mirror_softc *sc;
1033         struct g_mirror_disk *disk;
1034         struct bio *cbp;
1035         struct g_kerneldump *gkd;
1036
1037         /*
1038          * We configure dumping to the first component, because this component
1039          * will be used for reading with 'prefer' balance algorithm.
1040          * If the component with the higest priority is currently disconnected
1041          * we will not be able to read the dump after the reboot if it will be
1042          * connected and synchronized later. Can we do something better?
1043          */
1044         sc = bp->bio_to->geom->softc;
1045         disk = LIST_FIRST(&sc->sc_disks);
1046
1047         gkd = (struct g_kerneldump *)bp->bio_data;
1048         if (gkd->length > bp->bio_to->mediasize)
1049                 gkd->length = bp->bio_to->mediasize;
1050         cbp = g_clone_bio(bp);
1051         if (cbp == NULL) {
1052                 g_io_deliver(bp, ENOMEM);
1053                 return;
1054         }
1055         cbp->bio_done = g_std_done;
1056         g_io_request(cbp, disk->d_consumer);
1057         G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1058             g_mirror_get_diskname(disk));
1059 }
1060
1061 static void
1062 g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
1063 {
1064         struct bio_queue_head queue;
1065         struct g_mirror_disk *disk;
1066         struct g_consumer *cp;
1067         struct bio *cbp;
1068
1069         bioq_init(&queue);
1070         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1071                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1072                         continue;
1073                 cbp = g_clone_bio(bp);
1074                 if (cbp == NULL) {
1075                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1076                                 g_destroy_bio(cbp);
1077                         if (bp->bio_error == 0)
1078                                 bp->bio_error = ENOMEM;
1079                         g_io_deliver(bp, bp->bio_error);
1080                         return;
1081                 }
1082                 bioq_insert_tail(&queue, cbp);
1083                 cbp->bio_done = g_mirror_flush_done;
1084                 cbp->bio_caller1 = disk;
1085                 cbp->bio_to = disk->d_consumer->provider;
1086         }
1087         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1088                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1089                 disk = cbp->bio_caller1;
1090                 cbp->bio_caller1 = NULL;
1091                 cp = disk->d_consumer;
1092                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1093                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1094                     cp->acr, cp->acw, cp->ace));
1095                 g_io_request(cbp, disk->d_consumer);
1096         }
1097 }
1098
1099 static void
1100 g_mirror_start(struct bio *bp)
1101 {
1102         struct g_mirror_softc *sc;
1103
1104         sc = bp->bio_to->geom->softc;
1105         /*
1106          * If sc == NULL or there are no valid disks, provider's error
1107          * should be set and g_mirror_start() should not be called at all.
1108          */
1109         KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1110             ("Provider's error should be set (error=%d)(mirror=%s).",
1111             bp->bio_to->error, bp->bio_to->name));
1112         G_MIRROR_LOGREQ(3, bp, "Request received.");
1113
1114         switch (bp->bio_cmd) {
1115         case BIO_READ:
1116         case BIO_WRITE:
1117         case BIO_DELETE:
1118                 break;
1119         case BIO_FLUSH:
1120                 g_mirror_flush(sc, bp);
1121                 return;
1122         case BIO_GETATTR:
1123                 if (g_handleattr_int(bp, "GEOM::candelete", 1))
1124                         return;
1125                 else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1126                         g_mirror_kernel_dump(bp);
1127                         return;
1128                 }
1129                 /* FALLTHROUGH */
1130         default:
1131                 g_io_deliver(bp, EOPNOTSUPP);
1132                 return;
1133         }
1134         mtx_lock(&sc->sc_queue_mtx);
1135         bioq_disksort(&sc->sc_queue, bp);
1136         mtx_unlock(&sc->sc_queue_mtx);
1137         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1138         wakeup(sc);
1139 }
1140
1141 /*
1142  * Return TRUE if the given request is colliding with a in-progress
1143  * synchronization request.
1144  */
1145 static int
1146 g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1147 {
1148         struct g_mirror_disk *disk;
1149         struct bio *sbp;
1150         off_t rstart, rend, sstart, send;
1151         int i;
1152
1153         if (sc->sc_sync.ds_ndisks == 0)
1154                 return (0);
1155         rstart = bp->bio_offset;
1156         rend = bp->bio_offset + bp->bio_length;
1157         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1158                 if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1159                         continue;
1160                 for (i = 0; i < g_mirror_syncreqs; i++) {
1161                         sbp = disk->d_sync.ds_bios[i];
1162                         if (sbp == NULL)
1163                                 continue;
1164                         sstart = sbp->bio_offset;
1165                         send = sbp->bio_offset + sbp->bio_length;
1166                         if (rend > sstart && rstart < send)
1167                                 return (1);
1168                 }
1169         }
1170         return (0);
1171 }
1172
1173 /*
1174  * Return TRUE if the given sync request is colliding with a in-progress regular
1175  * request.
1176  */
1177 static int
1178 g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1179 {
1180         off_t rstart, rend, sstart, send;
1181         struct bio *bp;
1182
1183         if (sc->sc_sync.ds_ndisks == 0)
1184                 return (0);
1185         sstart = sbp->bio_offset;
1186         send = sbp->bio_offset + sbp->bio_length;
1187         TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1188                 rstart = bp->bio_offset;
1189                 rend = bp->bio_offset + bp->bio_length;
1190                 if (rend > sstart && rstart < send)
1191                         return (1);
1192         }
1193         return (0);
1194 }
1195
1196 /*
1197  * Puts request onto delayed queue.
1198  */
1199 static void
1200 g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1201 {
1202
1203         G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1204         bioq_insert_head(&sc->sc_regular_delayed, bp);
1205 }
1206
1207 /*
1208  * Puts synchronization request onto delayed queue.
1209  */
1210 static void
1211 g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1212 {
1213
1214         G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1215         bioq_insert_tail(&sc->sc_sync_delayed, bp);
1216 }
1217
1218 /*
1219  * Releases delayed regular requests which don't collide anymore with sync
1220  * requests.
1221  */
1222 static void
1223 g_mirror_regular_release(struct g_mirror_softc *sc)
1224 {
1225         struct bio *bp, *bp2;
1226
1227         TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1228                 if (g_mirror_sync_collision(sc, bp))
1229                         continue;
1230                 bioq_remove(&sc->sc_regular_delayed, bp);
1231                 G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1232                 mtx_lock(&sc->sc_queue_mtx);
1233                 bioq_insert_head(&sc->sc_queue, bp);
1234 #if 0
1235                 /*
1236                  * wakeup() is not needed, because this function is called from
1237                  * the worker thread.
1238                  */
1239                 wakeup(&sc->sc_queue);
1240 #endif
1241                 mtx_unlock(&sc->sc_queue_mtx);
1242         }
1243 }
1244
1245 /*
1246  * Releases delayed sync requests which don't collide anymore with regular
1247  * requests.
1248  */
1249 static void
1250 g_mirror_sync_release(struct g_mirror_softc *sc)
1251 {
1252         struct bio *bp, *bp2;
1253
1254         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1255                 if (g_mirror_regular_collision(sc, bp))
1256                         continue;
1257                 bioq_remove(&sc->sc_sync_delayed, bp);
1258                 G_MIRROR_LOGREQ(2, bp,
1259                     "Releasing delayed synchronization request.");
1260                 g_io_request(bp, bp->bio_from);
1261         }
1262 }
1263
1264 /*
1265  * Handle synchronization requests.
1266  * Every synchronization request is two-steps process: first, READ request is
1267  * send to active provider and then WRITE request (with read data) to the provider
1268  * beeing synchronized. When WRITE is finished, new synchronization request is
1269  * send.
1270  */
1271 static void
1272 g_mirror_sync_request(struct bio *bp)
1273 {
1274         struct g_mirror_softc *sc;
1275         struct g_mirror_disk *disk;
1276
1277         bp->bio_from->index--;
1278         sc = bp->bio_from->geom->softc;
1279         disk = bp->bio_from->private;
1280         if (disk == NULL) {
1281                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1282                 g_topology_lock();
1283                 g_mirror_kill_consumer(sc, bp->bio_from);
1284                 g_topology_unlock();
1285                 free(bp->bio_data, M_MIRROR);
1286                 g_destroy_bio(bp);
1287                 sx_xlock(&sc->sc_lock);
1288                 return;
1289         }
1290
1291         /*
1292          * Synchronization request.
1293          */
1294         switch (bp->bio_cmd) {
1295         case BIO_READ:
1296             {
1297                 struct g_consumer *cp;
1298
1299                 if (bp->bio_error != 0) {
1300                         G_MIRROR_LOGREQ(0, bp,
1301                             "Synchronization request failed (error=%d).",
1302                             bp->bio_error);
1303                         g_destroy_bio(bp);
1304                         return;
1305                 }
1306                 G_MIRROR_LOGREQ(3, bp,
1307                     "Synchronization request half-finished.");
1308                 bp->bio_cmd = BIO_WRITE;
1309                 bp->bio_cflags = 0;
1310                 cp = disk->d_consumer;
1311                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1312                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1313                     cp->acr, cp->acw, cp->ace));
1314                 cp->index++;
1315                 g_io_request(bp, cp);
1316                 return;
1317             }
1318         case BIO_WRITE:
1319             {
1320                 struct g_mirror_disk_sync *sync;
1321                 off_t offset;
1322                 void *data;
1323                 int i;
1324
1325                 if (bp->bio_error != 0) {
1326                         G_MIRROR_LOGREQ(0, bp,
1327                             "Synchronization request failed (error=%d).",
1328                             bp->bio_error);
1329                         g_destroy_bio(bp);
1330                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1331                         g_mirror_event_send(disk,
1332                             G_MIRROR_DISK_STATE_DISCONNECTED,
1333                             G_MIRROR_EVENT_DONTWAIT);
1334                         return;
1335                 }
1336                 G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1337                 sync = &disk->d_sync;
1338                 if (sync->ds_offset >= sc->sc_mediasize ||
1339                     sync->ds_consumer == NULL ||
1340                     (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1341                         /* Don't send more synchronization requests. */
1342                         sync->ds_inflight--;
1343                         if (sync->ds_bios != NULL) {
1344                                 i = (int)(uintptr_t)bp->bio_caller1;
1345                                 sync->ds_bios[i] = NULL;
1346                         }
1347                         free(bp->bio_data, M_MIRROR);
1348                         g_destroy_bio(bp);
1349                         if (sync->ds_inflight > 0)
1350                                 return;
1351                         if (sync->ds_consumer == NULL ||
1352                             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1353                                 return;
1354                         }
1355                         /* Disk up-to-date, activate it. */
1356                         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1357                             G_MIRROR_EVENT_DONTWAIT);
1358                         return;
1359                 }
1360
1361                 /* Send next synchronization request. */
1362                 data = bp->bio_data;
1363                 bzero(bp, sizeof(*bp));
1364                 bp->bio_cmd = BIO_READ;
1365                 bp->bio_offset = sync->ds_offset;
1366                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1367                 sync->ds_offset += bp->bio_length;
1368                 bp->bio_done = g_mirror_sync_done;
1369                 bp->bio_data = data;
1370                 bp->bio_from = sync->ds_consumer;
1371                 bp->bio_to = sc->sc_provider;
1372                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1373                 sync->ds_consumer->index++;
1374                 /*
1375                  * Delay the request if it is colliding with a regular request.
1376                  */
1377                 if (g_mirror_regular_collision(sc, bp))
1378                         g_mirror_sync_delay(sc, bp);
1379                 else
1380                         g_io_request(bp, sync->ds_consumer);
1381
1382                 /* Release delayed requests if possible. */
1383                 g_mirror_regular_release(sc);
1384
1385                 /* Find the smallest offset */
1386                 offset = sc->sc_mediasize;
1387                 for (i = 0; i < g_mirror_syncreqs; i++) {
1388                         bp = sync->ds_bios[i];
1389                         if (bp->bio_offset < offset)
1390                                 offset = bp->bio_offset;
1391                 }
1392                 if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
1393                         /* Update offset_done on every 100 blocks. */
1394                         sync->ds_offset_done = offset;
1395                         g_mirror_update_metadata(disk);
1396                 }
1397                 return;
1398             }
1399         default:
1400                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1401                     bp->bio_cmd, sc->sc_name));
1402                 break;
1403         }
1404 }
1405
1406 static void
1407 g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1408 {
1409         struct g_mirror_disk *disk;
1410         struct g_consumer *cp;
1411         struct bio *cbp;
1412
1413         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1414                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1415                         break;
1416         }
1417         if (disk == NULL) {
1418                 if (bp->bio_error == 0)
1419                         bp->bio_error = ENXIO;
1420                 g_io_deliver(bp, bp->bio_error);
1421                 return;
1422         }
1423         cbp = g_clone_bio(bp);
1424         if (cbp == NULL) {
1425                 if (bp->bio_error == 0)
1426                         bp->bio_error = ENOMEM;
1427                 g_io_deliver(bp, bp->bio_error);
1428                 return;
1429         }
1430         /*
1431          * Fill in the component buf structure.
1432          */
1433         cp = disk->d_consumer;
1434         cbp->bio_done = g_mirror_done;
1435         cbp->bio_to = cp->provider;
1436         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1437         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1438             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1439             cp->acw, cp->ace));
1440         cp->index++;
1441         g_io_request(cbp, cp);
1442 }
1443
1444 static void
1445 g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1446 {
1447         struct g_mirror_disk *disk;
1448         struct g_consumer *cp;
1449         struct bio *cbp;
1450
1451         disk = g_mirror_get_disk(sc);
1452         if (disk == NULL) {
1453                 if (bp->bio_error == 0)
1454                         bp->bio_error = ENXIO;
1455                 g_io_deliver(bp, bp->bio_error);
1456                 return;
1457         }
1458         cbp = g_clone_bio(bp);
1459         if (cbp == NULL) {
1460                 if (bp->bio_error == 0)
1461                         bp->bio_error = ENOMEM;
1462                 g_io_deliver(bp, bp->bio_error);
1463                 return;
1464         }
1465         /*
1466          * Fill in the component buf structure.
1467          */
1468         cp = disk->d_consumer;
1469         cbp->bio_done = g_mirror_done;
1470         cbp->bio_to = cp->provider;
1471         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1472         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1473             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1474             cp->acw, cp->ace));
1475         cp->index++;
1476         g_io_request(cbp, cp);
1477 }
1478
1479 #define TRACK_SIZE  (1 * 1024 * 1024)
1480 #define LOAD_SCALE      256
1481 #define ABS(x)          (((x) >= 0) ? (x) : (-(x)))
1482
1483 static void
1484 g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1485 {
1486         struct g_mirror_disk *disk, *dp;
1487         struct g_consumer *cp;
1488         struct bio *cbp;
1489         int prio, best;
1490
1491         /* Find a disk with the smallest load. */
1492         disk = NULL;
1493         best = INT_MAX;
1494         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1495                 if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1496                         continue;
1497                 prio = dp->load;
1498                 /* If disk head is precisely in position - highly prefer it. */
1499                 if (dp->d_last_offset == bp->bio_offset)
1500                         prio -= 2 * LOAD_SCALE;
1501                 else
1502                 /* If disk head is close to position - prefer it. */
1503                 if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1504                         prio -= 1 * LOAD_SCALE;
1505                 if (prio <= best) {
1506                         disk = dp;
1507                         best = prio;
1508                 }
1509         }
1510         KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1511         cbp = g_clone_bio(bp);
1512         if (cbp == NULL) {
1513                 if (bp->bio_error == 0)
1514                         bp->bio_error = ENOMEM;
1515                 g_io_deliver(bp, bp->bio_error);
1516                 return;
1517         }
1518         /*
1519          * Fill in the component buf structure.
1520          */
1521         cp = disk->d_consumer;
1522         cbp->bio_done = g_mirror_done;
1523         cbp->bio_to = cp->provider;
1524         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1525         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1526             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1527             cp->acw, cp->ace));
1528         cp->index++;
1529         /* Remember last head position */
1530         disk->d_last_offset = bp->bio_offset + bp->bio_length;
1531         /* Update loads. */
1532         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1533                 dp->load = (dp->d_consumer->index * LOAD_SCALE +
1534                     dp->load * 7) / 8;
1535         }
1536         g_io_request(cbp, cp);
1537 }
1538
1539 static void
1540 g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1541 {
1542         struct bio_queue_head queue;
1543         struct g_mirror_disk *disk;
1544         struct g_consumer *cp;
1545         struct bio *cbp;
1546         off_t left, mod, offset, slice;
1547         u_char *data;
1548         u_int ndisks;
1549
1550         if (bp->bio_length <= sc->sc_slice) {
1551                 g_mirror_request_round_robin(sc, bp);
1552                 return;
1553         }
1554         ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1555         slice = bp->bio_length / ndisks;
1556         mod = slice % sc->sc_provider->sectorsize;
1557         if (mod != 0)
1558                 slice += sc->sc_provider->sectorsize - mod;
1559         /*
1560          * Allocate all bios before sending any request, so we can
1561          * return ENOMEM in nice and clean way.
1562          */
1563         left = bp->bio_length;
1564         offset = bp->bio_offset;
1565         data = bp->bio_data;
1566         bioq_init(&queue);
1567         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1568                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1569                         continue;
1570                 cbp = g_clone_bio(bp);
1571                 if (cbp == NULL) {
1572                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1573                                 bioq_remove(&queue, cbp);
1574                         if (bp->bio_error == 0)
1575                                 bp->bio_error = ENOMEM;
1576                         g_io_deliver(bp, bp->bio_error);
1577                         return;
1578                 }
1579                 bioq_insert_tail(&queue, cbp);
1580                 cbp->bio_done = g_mirror_done;
1581                 cbp->bio_caller1 = disk;
1582                 cbp->bio_to = disk->d_consumer->provider;
1583                 cbp->bio_offset = offset;
1584                 cbp->bio_data = data;
1585                 cbp->bio_length = MIN(left, slice);
1586                 left -= cbp->bio_length;
1587                 if (left == 0)
1588                         break;
1589                 offset += cbp->bio_length;
1590                 data += cbp->bio_length;
1591         }
1592         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1593                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1594                 disk = cbp->bio_caller1;
1595                 cbp->bio_caller1 = NULL;
1596                 cp = disk->d_consumer;
1597                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1598                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1599                     cp->acr, cp->acw, cp->ace));
1600                 disk->d_consumer->index++;
1601                 g_io_request(cbp, disk->d_consumer);
1602         }
1603 }
1604
1605 static void
1606 g_mirror_register_request(struct bio *bp)
1607 {
1608         struct g_mirror_softc *sc;
1609
1610         sc = bp->bio_to->geom->softc;
1611         switch (bp->bio_cmd) {
1612         case BIO_READ:
1613                 switch (sc->sc_balance) {
1614                 case G_MIRROR_BALANCE_LOAD:
1615                         g_mirror_request_load(sc, bp);
1616                         break;
1617                 case G_MIRROR_BALANCE_PREFER:
1618                         g_mirror_request_prefer(sc, bp);
1619                         break;
1620                 case G_MIRROR_BALANCE_ROUND_ROBIN:
1621                         g_mirror_request_round_robin(sc, bp);
1622                         break;
1623                 case G_MIRROR_BALANCE_SPLIT:
1624                         g_mirror_request_split(sc, bp);
1625                         break;
1626                 }
1627                 return;
1628         case BIO_WRITE:
1629         case BIO_DELETE:
1630             {
1631                 struct g_mirror_disk *disk;
1632                 struct g_mirror_disk_sync *sync;
1633                 struct bio_queue_head queue;
1634                 struct g_consumer *cp;
1635                 struct bio *cbp;
1636
1637                 /*
1638                  * Delay the request if it is colliding with a synchronization
1639                  * request.
1640                  */
1641                 if (g_mirror_sync_collision(sc, bp)) {
1642                         g_mirror_regular_delay(sc, bp);
1643                         return;
1644                 }
1645
1646                 if (sc->sc_idle)
1647                         g_mirror_unidle(sc);
1648                 else
1649                         sc->sc_last_write = time_uptime;
1650
1651                 /*
1652                  * Allocate all bios before sending any request, so we can
1653                  * return ENOMEM in nice and clean way.
1654                  */
1655                 bioq_init(&queue);
1656                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1657                         sync = &disk->d_sync;
1658                         switch (disk->d_state) {
1659                         case G_MIRROR_DISK_STATE_ACTIVE:
1660                                 break;
1661                         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1662                                 if (bp->bio_offset >= sync->ds_offset)
1663                                         continue;
1664                                 break;
1665                         default:
1666                                 continue;
1667                         }
1668                         if (bp->bio_cmd == BIO_DELETE &&
1669                             (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1670                                 continue;
1671                         cbp = g_clone_bio(bp);
1672                         if (cbp == NULL) {
1673                                 while ((cbp = bioq_takefirst(&queue)) != NULL)
1674                                         g_destroy_bio(cbp);
1675                                 if (bp->bio_error == 0)
1676                                         bp->bio_error = ENOMEM;
1677                                 g_io_deliver(bp, bp->bio_error);
1678                                 return;
1679                         }
1680                         bioq_insert_tail(&queue, cbp);
1681                         cbp->bio_done = g_mirror_done;
1682                         cp = disk->d_consumer;
1683                         cbp->bio_caller1 = cp;
1684                         cbp->bio_to = cp->provider;
1685                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1686                             ("Consumer %s not opened (r%dw%de%d).",
1687                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1688                 }
1689                 while ((cbp = bioq_takefirst(&queue)) != NULL) {
1690                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1691                         cp = cbp->bio_caller1;
1692                         cbp->bio_caller1 = NULL;
1693                         cp->index++;
1694                         sc->sc_writes++;
1695                         g_io_request(cbp, cp);
1696                 }
1697                 /*
1698                  * Put request onto inflight queue, so we can check if new
1699                  * synchronization requests don't collide with it.
1700                  */
1701                 bioq_insert_tail(&sc->sc_inflight, bp);
1702                 /*
1703                  * Bump syncid on first write.
1704                  */
1705                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1706                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1707                         g_mirror_bump_syncid(sc);
1708                 }
1709                 return;
1710             }
1711         default:
1712                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1713                     bp->bio_cmd, sc->sc_name));
1714                 break;
1715         }
1716 }
1717
1718 static int
1719 g_mirror_can_destroy(struct g_mirror_softc *sc)
1720 {
1721         struct g_geom *gp;
1722         struct g_consumer *cp;
1723
1724         g_topology_assert();
1725         gp = sc->sc_geom;
1726         if (gp->softc == NULL)
1727                 return (1);
1728         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1729                 return (0);
1730         LIST_FOREACH(cp, &gp->consumer, consumer) {
1731                 if (g_mirror_is_busy(sc, cp))
1732                         return (0);
1733         }
1734         gp = sc->sc_sync.ds_geom;
1735         LIST_FOREACH(cp, &gp->consumer, consumer) {
1736                 if (g_mirror_is_busy(sc, cp))
1737                         return (0);
1738         }
1739         G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1740             sc->sc_name);
1741         return (1);
1742 }
1743
1744 static int
1745 g_mirror_try_destroy(struct g_mirror_softc *sc)
1746 {
1747
1748         if (sc->sc_rootmount != NULL) {
1749                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1750                     sc->sc_rootmount);
1751                 root_mount_rel(sc->sc_rootmount);
1752                 sc->sc_rootmount = NULL;
1753         }
1754         g_topology_lock();
1755         if (!g_mirror_can_destroy(sc)) {
1756                 g_topology_unlock();
1757                 return (0);
1758         }
1759         sc->sc_geom->softc = NULL;
1760         sc->sc_sync.ds_geom->softc = NULL;
1761         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
1762                 g_topology_unlock();
1763                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1764                     &sc->sc_worker);
1765                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
1766                 sx_xunlock(&sc->sc_lock);
1767                 wakeup(&sc->sc_worker);
1768                 sc->sc_worker = NULL;
1769         } else {
1770                 g_topology_unlock();
1771                 g_mirror_destroy_device(sc);
1772                 free(sc, M_MIRROR);
1773         }
1774         return (1);
1775 }
1776
1777 /*
1778  * Worker thread.
1779  */
1780 static void
1781 g_mirror_worker(void *arg)
1782 {
1783         struct g_mirror_softc *sc;
1784         struct g_mirror_event *ep;
1785         struct bio *bp;
1786         int timeout;
1787
1788         sc = arg;
1789         thread_lock(curthread);
1790         sched_prio(curthread, PRIBIO);
1791         thread_unlock(curthread);
1792
1793         sx_xlock(&sc->sc_lock);
1794         for (;;) {
1795                 G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1796                 /*
1797                  * First take a look at events.
1798                  * This is important to handle events before any I/O requests.
1799                  */
1800                 ep = g_mirror_event_get(sc);
1801                 if (ep != NULL) {
1802                         g_mirror_event_remove(sc, ep);
1803                         if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1804                                 /* Update only device status. */
1805                                 G_MIRROR_DEBUG(3,
1806                                     "Running event for device %s.",
1807                                     sc->sc_name);
1808                                 ep->e_error = 0;
1809                                 g_mirror_update_device(sc, 1);
1810                         } else {
1811                                 /* Update disk status. */
1812                                 G_MIRROR_DEBUG(3, "Running event for disk %s.",
1813                                      g_mirror_get_diskname(ep->e_disk));
1814                                 ep->e_error = g_mirror_update_disk(ep->e_disk,
1815                                     ep->e_state);
1816                                 if (ep->e_error == 0)
1817                                         g_mirror_update_device(sc, 0);
1818                         }
1819                         if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1820                                 KASSERT(ep->e_error == 0,
1821                                     ("Error cannot be handled."));
1822                                 g_mirror_event_free(ep);
1823                         } else {
1824                                 ep->e_flags |= G_MIRROR_EVENT_DONE;
1825                                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1826                                     ep);
1827                                 mtx_lock(&sc->sc_events_mtx);
1828                                 wakeup(ep);
1829                                 mtx_unlock(&sc->sc_events_mtx);
1830                         }
1831                         if ((sc->sc_flags &
1832                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1833                                 if (g_mirror_try_destroy(sc)) {
1834                                         curthread->td_pflags &= ~TDP_GEOM;
1835                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1836                                         kproc_exit(0);
1837                                 }
1838                         }
1839                         G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1840                         continue;
1841                 }
1842                 /*
1843                  * Check if we can mark array as CLEAN and if we can't take
1844                  * how much seconds should we wait.
1845                  */
1846                 timeout = g_mirror_idle(sc, -1);
1847                 /*
1848                  * Now I/O requests.
1849                  */
1850                 /* Get first request from the queue. */
1851                 mtx_lock(&sc->sc_queue_mtx);
1852                 bp = bioq_first(&sc->sc_queue);
1853                 if (bp == NULL) {
1854                         if ((sc->sc_flags &
1855                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1856                                 mtx_unlock(&sc->sc_queue_mtx);
1857                                 if (g_mirror_try_destroy(sc)) {
1858                                         curthread->td_pflags &= ~TDP_GEOM;
1859                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1860                                         kproc_exit(0);
1861                                 }
1862                                 mtx_lock(&sc->sc_queue_mtx);
1863                         }
1864                         sx_xunlock(&sc->sc_lock);
1865                         /*
1866                          * XXX: We can miss an event here, because an event
1867                          *      can be added without sx-device-lock and without
1868                          *      mtx-queue-lock. Maybe I should just stop using
1869                          *      dedicated mutex for events synchronization and
1870                          *      stick with the queue lock?
1871                          *      The event will hang here until next I/O request
1872                          *      or next event is received.
1873                          */
1874                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1875                             timeout * hz);
1876                         sx_xlock(&sc->sc_lock);
1877                         G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1878                         continue;
1879                 }
1880                 bioq_remove(&sc->sc_queue, bp);
1881                 mtx_unlock(&sc->sc_queue_mtx);
1882
1883                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
1884                     (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
1885                         g_mirror_sync_request(bp);      /* READ */
1886                 } else if (bp->bio_to != sc->sc_provider) {
1887                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
1888                                 g_mirror_regular_request(bp);
1889                         else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
1890                                 g_mirror_sync_request(bp);      /* WRITE */
1891                         else {
1892                                 KASSERT(0,
1893                                     ("Invalid request cflags=0x%hhx to=%s.",
1894                                     bp->bio_cflags, bp->bio_to->name));
1895                         }
1896                 } else {
1897                         g_mirror_register_request(bp);
1898                 }
1899                 G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
1900         }
1901 }
1902
1903 static void
1904 g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
1905 {
1906
1907         sx_assert(&sc->sc_lock, SX_LOCKED);
1908
1909         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
1910                 return;
1911         if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
1912                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
1913                     g_mirror_get_diskname(disk), sc->sc_name);
1914                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1915         } else if (sc->sc_idle &&
1916             (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
1917                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
1918                     g_mirror_get_diskname(disk), sc->sc_name);
1919                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
1920         }
1921 }
1922
1923 static void
1924 g_mirror_sync_start(struct g_mirror_disk *disk)
1925 {
1926         struct g_mirror_softc *sc;
1927         struct g_consumer *cp;
1928         struct bio *bp;
1929         int error, i;
1930
1931         g_topology_assert_not();
1932         sc = disk->d_softc;
1933         sx_assert(&sc->sc_lock, SX_LOCKED);
1934
1935         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
1936             ("Disk %s is not marked for synchronization.",
1937             g_mirror_get_diskname(disk)));
1938         KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1939             ("Device not in RUNNING state (%s, %u).", sc->sc_name,
1940             sc->sc_state));
1941
1942         sx_xunlock(&sc->sc_lock);
1943         g_topology_lock();
1944         cp = g_new_consumer(sc->sc_sync.ds_geom);
1945         cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
1946         error = g_attach(cp, sc->sc_provider);
1947         KASSERT(error == 0,
1948             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
1949         error = g_access(cp, 1, 0, 0);
1950         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
1951         g_topology_unlock();
1952         sx_xlock(&sc->sc_lock);
1953
1954         G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
1955             g_mirror_get_diskname(disk));
1956         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
1957                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1958         KASSERT(disk->d_sync.ds_consumer == NULL,
1959             ("Sync consumer already exists (device=%s, disk=%s).",
1960             sc->sc_name, g_mirror_get_diskname(disk)));
1961
1962         disk->d_sync.ds_consumer = cp;
1963         disk->d_sync.ds_consumer->private = disk;
1964         disk->d_sync.ds_consumer->index = 0;
1965
1966         /*
1967          * Allocate memory for synchronization bios and initialize them.
1968          */
1969         disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
1970             M_MIRROR, M_WAITOK);
1971         for (i = 0; i < g_mirror_syncreqs; i++) {
1972                 bp = g_alloc_bio();
1973                 disk->d_sync.ds_bios[i] = bp;
1974                 bp->bio_parent = NULL;
1975                 bp->bio_cmd = BIO_READ;
1976                 bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
1977                 bp->bio_cflags = 0;
1978                 bp->bio_offset = disk->d_sync.ds_offset;
1979                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1980                 disk->d_sync.ds_offset += bp->bio_length;
1981                 bp->bio_done = g_mirror_sync_done;
1982                 bp->bio_from = disk->d_sync.ds_consumer;
1983                 bp->bio_to = sc->sc_provider;
1984                 bp->bio_caller1 = (void *)(uintptr_t)i;
1985         }
1986
1987         /* Increase the number of disks in SYNCHRONIZING state. */
1988         sc->sc_sync.ds_ndisks++;
1989         /* Set the number of in-flight synchronization requests. */
1990         disk->d_sync.ds_inflight = g_mirror_syncreqs;
1991
1992         /*
1993          * Fire off first synchronization requests.
1994          */
1995         for (i = 0; i < g_mirror_syncreqs; i++) {
1996                 bp = disk->d_sync.ds_bios[i];
1997                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1998                 disk->d_sync.ds_consumer->index++;
1999                 /*
2000                  * Delay the request if it is colliding with a regular request.
2001                  */
2002                 if (g_mirror_regular_collision(sc, bp))
2003                         g_mirror_sync_delay(sc, bp);
2004                 else
2005                         g_io_request(bp, disk->d_sync.ds_consumer);
2006         }
2007 }
2008
2009 /*
2010  * Stop synchronization process.
2011  * type: 0 - synchronization finished
2012  *       1 - synchronization stopped
2013  */
2014 static void
2015 g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2016 {
2017         struct g_mirror_softc *sc;
2018         struct g_consumer *cp;
2019
2020         g_topology_assert_not();
2021         sc = disk->d_softc;
2022         sx_assert(&sc->sc_lock, SX_LOCKED);
2023
2024         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2025             ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2026             g_mirror_disk_state2str(disk->d_state)));
2027         if (disk->d_sync.ds_consumer == NULL)
2028                 return;
2029
2030         if (type == 0) {
2031                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2032                     sc->sc_name, g_mirror_get_diskname(disk));
2033         } else /* if (type == 1) */ {
2034                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2035                     sc->sc_name, g_mirror_get_diskname(disk));
2036         }
2037         free(disk->d_sync.ds_bios, M_MIRROR);
2038         disk->d_sync.ds_bios = NULL;
2039         cp = disk->d_sync.ds_consumer;
2040         disk->d_sync.ds_consumer = NULL;
2041         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2042         sc->sc_sync.ds_ndisks--;
2043         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2044         g_topology_lock();
2045         g_mirror_kill_consumer(sc, cp);
2046         g_topology_unlock();
2047         sx_xlock(&sc->sc_lock);
2048 }
2049
2050 static void
2051 g_mirror_launch_provider(struct g_mirror_softc *sc)
2052 {
2053         struct g_mirror_disk *disk;
2054         struct g_provider *pp, *dp;
2055
2056         sx_assert(&sc->sc_lock, SX_LOCKED);
2057
2058         g_topology_lock();
2059         pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2060         pp->flags |= G_PF_DIRECT_RECEIVE;
2061         pp->mediasize = sc->sc_mediasize;
2062         pp->sectorsize = sc->sc_sectorsize;
2063         pp->stripesize = 0;
2064         pp->stripeoffset = 0;
2065
2066         /* Splitting of unmapped BIO's could work but isn't implemented now */
2067         if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2068                 pp->flags |= G_PF_ACCEPT_UNMAPPED;
2069
2070         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2071                 if (disk->d_consumer && disk->d_consumer->provider) {
2072                         dp = disk->d_consumer->provider;
2073                         if (dp->stripesize > pp->stripesize) {
2074                                 pp->stripesize = dp->stripesize;
2075                                 pp->stripeoffset = dp->stripeoffset;
2076                         }
2077                         /* A provider underneath us doesn't support unmapped */
2078                         if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2079                                 G_MIRROR_DEBUG(0, "Cancelling unmapped "
2080                                     "because of %s.", dp->name);
2081                                 pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2082                         }
2083                 }
2084         }
2085         sc->sc_provider = pp;
2086         g_error_provider(pp, 0);
2087         g_topology_unlock();
2088         G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2089             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2090         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2091                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2092                         g_mirror_sync_start(disk);
2093         }
2094 }
2095
2096 static void
2097 g_mirror_destroy_provider(struct g_mirror_softc *sc)
2098 {
2099         struct g_mirror_disk *disk;
2100         struct bio *bp;
2101
2102         g_topology_assert_not();
2103         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2104             sc->sc_name));
2105
2106         g_topology_lock();
2107         g_error_provider(sc->sc_provider, ENXIO);
2108         mtx_lock(&sc->sc_queue_mtx);
2109         while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL)
2110                 g_io_deliver(bp, ENXIO);
2111         mtx_unlock(&sc->sc_queue_mtx);
2112         G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2113             sc->sc_provider->name);
2114         sc->sc_provider->flags |= G_PF_WITHER;
2115         g_orphan_provider(sc->sc_provider, ENXIO);
2116         g_topology_unlock();
2117         sc->sc_provider = NULL;
2118         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2119                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2120                         g_mirror_sync_stop(disk, 1);
2121         }
2122 }
2123
2124 static void
2125 g_mirror_go(void *arg)
2126 {
2127         struct g_mirror_softc *sc;
2128
2129         sc = arg;
2130         G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2131         g_mirror_event_send(sc, 0,
2132             G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2133 }
2134
2135 static u_int
2136 g_mirror_determine_state(struct g_mirror_disk *disk)
2137 {
2138         struct g_mirror_softc *sc;
2139         u_int state;
2140
2141         sc = disk->d_softc;
2142         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2143                 if ((disk->d_flags &
2144                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2145                         /* Disk does not need synchronization. */
2146                         state = G_MIRROR_DISK_STATE_ACTIVE;
2147                 } else {
2148                         if ((sc->sc_flags &
2149                              G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2150                             (disk->d_flags &
2151                              G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2152                                 /*
2153                                  * We can start synchronization from
2154                                  * the stored offset.
2155                                  */
2156                                 state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2157                         } else {
2158                                 state = G_MIRROR_DISK_STATE_STALE;
2159                         }
2160                 }
2161         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2162                 /*
2163                  * Reset all synchronization data for this disk,
2164                  * because if it even was synchronized, it was
2165                  * synchronized to disks with different syncid.
2166                  */
2167                 disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2168                 disk->d_sync.ds_offset = 0;
2169                 disk->d_sync.ds_offset_done = 0;
2170                 disk->d_sync.ds_syncid = sc->sc_syncid;
2171                 if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2172                     (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2173                         state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2174                 } else {
2175                         state = G_MIRROR_DISK_STATE_STALE;
2176                 }
2177         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2178                 /*
2179                  * Not good, NOT GOOD!
2180                  * It means that mirror was started on stale disks
2181                  * and more fresh disk just arrive.
2182                  * If there were writes, mirror is broken, sorry.
2183                  * I think the best choice here is don't touch
2184                  * this disk and inform the user loudly.
2185                  */
2186                 G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2187                     "disk (%s) arrives!! It will not be connected to the "
2188                     "running device.", sc->sc_name,
2189                     g_mirror_get_diskname(disk));
2190                 g_mirror_destroy_disk(disk);
2191                 state = G_MIRROR_DISK_STATE_NONE;
2192                 /* Return immediately, because disk was destroyed. */
2193                 return (state);
2194         }
2195         G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2196             g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2197         return (state);
2198 }
2199
2200 /*
2201  * Update device state.
2202  */
2203 static void
2204 g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
2205 {
2206         struct g_mirror_disk *disk;
2207         u_int state;
2208
2209         sx_assert(&sc->sc_lock, SX_XLOCKED);
2210
2211         switch (sc->sc_state) {
2212         case G_MIRROR_DEVICE_STATE_STARTING:
2213             {
2214                 struct g_mirror_disk *pdisk, *tdisk;
2215                 u_int dirty, ndisks, genid, syncid;
2216
2217                 KASSERT(sc->sc_provider == NULL,
2218                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2219                 /*
2220                  * Are we ready? We are, if all disks are connected or
2221                  * if we have any disks and 'force' is true.
2222                  */
2223                 ndisks = g_mirror_ndisks(sc, -1);
2224                 if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
2225                         ;
2226                 } else if (ndisks == 0) {
2227                         /*
2228                          * Disks went down in starting phase, so destroy
2229                          * device.
2230                          */
2231                         callout_drain(&sc->sc_callout);
2232                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2233                         G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2234                             sc->sc_rootmount);
2235                         root_mount_rel(sc->sc_rootmount);
2236                         sc->sc_rootmount = NULL;
2237                         return;
2238                 } else {
2239                         return;
2240                 }
2241
2242                 /*
2243                  * Activate all disks with the biggest syncid.
2244                  */
2245                 if (force) {
2246                         /*
2247                          * If 'force' is true, we have been called due to
2248                          * timeout, so don't bother canceling timeout.
2249                          */
2250                         ndisks = 0;
2251                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2252                                 if ((disk->d_flags &
2253                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2254                                         ndisks++;
2255                                 }
2256                         }
2257                         if (ndisks == 0) {
2258                                 /* No valid disks found, destroy device. */
2259                                 sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2260                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2261                                     __LINE__, sc->sc_rootmount);
2262                                 root_mount_rel(sc->sc_rootmount);
2263                                 sc->sc_rootmount = NULL;
2264                                 return;
2265                         }
2266                 } else {
2267                         /* Cancel timeout. */
2268                         callout_drain(&sc->sc_callout);
2269                 }
2270
2271                 /*
2272                  * Find the biggest genid.
2273                  */
2274                 genid = 0;
2275                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2276                         if (disk->d_genid > genid)
2277                                 genid = disk->d_genid;
2278                 }
2279                 sc->sc_genid = genid;
2280                 /*
2281                  * Remove all disks without the biggest genid.
2282                  */
2283                 LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2284                         if (disk->d_genid < genid) {
2285                                 G_MIRROR_DEBUG(0,
2286                                     "Component %s (device %s) broken, skipping.",
2287                                     g_mirror_get_diskname(disk), sc->sc_name);
2288                                 g_mirror_destroy_disk(disk);
2289                         }
2290                 }
2291
2292                 /*
2293                  * Find the biggest syncid.
2294                  */
2295                 syncid = 0;
2296                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2297                         if (disk->d_sync.ds_syncid > syncid)
2298                                 syncid = disk->d_sync.ds_syncid;
2299                 }
2300
2301                 /*
2302                  * Here we need to look for dirty disks and if all disks
2303                  * with the biggest syncid are dirty, we have to choose
2304                  * one with the biggest priority and rebuild the rest.
2305                  */
2306                 /*
2307                  * Find the number of dirty disks with the biggest syncid.
2308                  * Find the number of disks with the biggest syncid.
2309                  * While here, find a disk with the biggest priority.
2310                  */
2311                 dirty = ndisks = 0;
2312                 pdisk = NULL;
2313                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2314                         if (disk->d_sync.ds_syncid != syncid)
2315                                 continue;
2316                         if ((disk->d_flags &
2317                             G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2318                                 continue;
2319                         }
2320                         ndisks++;
2321                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2322                                 dirty++;
2323                                 if (pdisk == NULL ||
2324                                     pdisk->d_priority < disk->d_priority) {
2325                                         pdisk = disk;
2326                                 }
2327                         }
2328                 }
2329                 if (dirty == 0) {
2330                         /* No dirty disks at all, great. */
2331                 } else if (dirty == ndisks) {
2332                         /*
2333                          * Force synchronization for all dirty disks except one
2334                          * with the biggest priority.
2335                          */
2336                         KASSERT(pdisk != NULL, ("pdisk == NULL"));
2337                         G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2338                             "master disk for synchronization.",
2339                             g_mirror_get_diskname(pdisk), sc->sc_name);
2340                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2341                                 if (disk->d_sync.ds_syncid != syncid)
2342                                         continue;
2343                                 if ((disk->d_flags &
2344                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2345                                         continue;
2346                                 }
2347                                 KASSERT((disk->d_flags &
2348                                     G_MIRROR_DISK_FLAG_DIRTY) != 0,
2349                                     ("Disk %s isn't marked as dirty.",
2350                                     g_mirror_get_diskname(disk)));
2351                                 /* Skip the disk with the biggest priority. */
2352                                 if (disk == pdisk)
2353                                         continue;
2354                                 disk->d_sync.ds_syncid = 0;
2355                         }
2356                 } else if (dirty < ndisks) {
2357                         /*
2358                          * Force synchronization for all dirty disks.
2359                          * We have some non-dirty disks.
2360                          */
2361                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2362                                 if (disk->d_sync.ds_syncid != syncid)
2363                                         continue;
2364                                 if ((disk->d_flags &
2365                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2366                                         continue;
2367                                 }
2368                                 if ((disk->d_flags &
2369                                     G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2370                                         continue;
2371                                 }
2372                                 disk->d_sync.ds_syncid = 0;
2373                         }
2374                 }
2375
2376                 /* Reset hint. */
2377                 sc->sc_hint = NULL;
2378                 sc->sc_syncid = syncid;
2379                 if (force) {
2380                         /* Remember to bump syncid on first write. */
2381                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2382                 }
2383                 state = G_MIRROR_DEVICE_STATE_RUNNING;
2384                 G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2385                     sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2386                     g_mirror_device_state2str(state));
2387                 sc->sc_state = state;
2388                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2389                         state = g_mirror_determine_state(disk);
2390                         g_mirror_event_send(disk, state,
2391                             G_MIRROR_EVENT_DONTWAIT);
2392                         if (state == G_MIRROR_DISK_STATE_STALE)
2393                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2394                 }
2395                 break;
2396             }
2397         case G_MIRROR_DEVICE_STATE_RUNNING:
2398                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2399                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2400                         /*
2401                          * No active disks or no disks at all,
2402                          * so destroy device.
2403                          */
2404                         if (sc->sc_provider != NULL)
2405                                 g_mirror_destroy_provider(sc);
2406                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2407                         break;
2408                 } else if (g_mirror_ndisks(sc,
2409                     G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2410                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2411                         /*
2412                          * We have active disks, launch provider if it doesn't
2413                          * exist.
2414                          */
2415                         if (sc->sc_provider == NULL)
2416                                 g_mirror_launch_provider(sc);
2417                         if (sc->sc_rootmount != NULL) {
2418                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2419                                     __LINE__, sc->sc_rootmount);
2420                                 root_mount_rel(sc->sc_rootmount);
2421                                 sc->sc_rootmount = NULL;
2422                         }
2423                 }
2424                 /*
2425                  * Genid should be bumped immediately, so do it here.
2426                  */
2427                 if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2428                         sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2429                         g_mirror_bump_genid(sc);
2430                 }
2431                 break;
2432         default:
2433                 KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2434                     sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2435                 break;
2436         }
2437 }
2438
2439 /*
2440  * Update disk state and device state if needed.
2441  */
2442 #define DISK_STATE_CHANGED()    G_MIRROR_DEBUG(1,                       \
2443         "Disk %s state changed from %s to %s (device %s).",             \
2444         g_mirror_get_diskname(disk),                                    \
2445         g_mirror_disk_state2str(disk->d_state),                         \
2446         g_mirror_disk_state2str(state), sc->sc_name)
2447 static int
2448 g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2449 {
2450         struct g_mirror_softc *sc;
2451
2452         sc = disk->d_softc;
2453         sx_assert(&sc->sc_lock, SX_XLOCKED);
2454
2455 again:
2456         G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2457             g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2458             g_mirror_disk_state2str(state));
2459         switch (state) {
2460         case G_MIRROR_DISK_STATE_NEW:
2461                 /*
2462                  * Possible scenarios:
2463                  * 1. New disk arrive.
2464                  */
2465                 /* Previous state should be NONE. */
2466                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2467                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2468                     g_mirror_disk_state2str(disk->d_state)));
2469                 DISK_STATE_CHANGED();
2470
2471                 disk->d_state = state;
2472                 if (LIST_EMPTY(&sc->sc_disks))
2473                         LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2474                 else {
2475                         struct g_mirror_disk *dp;
2476
2477                         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2478                                 if (disk->d_priority >= dp->d_priority) {
2479                                         LIST_INSERT_BEFORE(dp, disk, d_next);
2480                                         dp = NULL;
2481                                         break;
2482                                 }
2483                                 if (LIST_NEXT(dp, d_next) == NULL)
2484                                         break;
2485                         }
2486                         if (dp != NULL)
2487                                 LIST_INSERT_AFTER(dp, disk, d_next);
2488                 }
2489                 G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2490                     sc->sc_name, g_mirror_get_diskname(disk));
2491                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2492                         break;
2493                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2494                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2495                     g_mirror_device_state2str(sc->sc_state),
2496                     g_mirror_get_diskname(disk),
2497                     g_mirror_disk_state2str(disk->d_state)));
2498                 state = g_mirror_determine_state(disk);
2499                 if (state != G_MIRROR_DISK_STATE_NONE)
2500                         goto again;
2501                 break;
2502         case G_MIRROR_DISK_STATE_ACTIVE:
2503                 /*
2504                  * Possible scenarios:
2505                  * 1. New disk does not need synchronization.
2506                  * 2. Synchronization process finished successfully.
2507                  */
2508                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2509                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2510                     g_mirror_device_state2str(sc->sc_state),
2511                     g_mirror_get_diskname(disk),
2512                     g_mirror_disk_state2str(disk->d_state)));
2513                 /* Previous state should be NEW or SYNCHRONIZING. */
2514                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2515                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2516                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2517                     g_mirror_disk_state2str(disk->d_state)));
2518                 DISK_STATE_CHANGED();
2519
2520                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2521                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2522                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2523                         g_mirror_sync_stop(disk, 0);
2524                 }
2525                 disk->d_state = state;
2526                 disk->d_sync.ds_offset = 0;
2527                 disk->d_sync.ds_offset_done = 0;
2528                 g_mirror_update_idle(sc, disk);
2529                 g_mirror_update_metadata(disk);
2530                 G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2531                     sc->sc_name, g_mirror_get_diskname(disk));
2532                 break;
2533         case G_MIRROR_DISK_STATE_STALE:
2534                 /*
2535                  * Possible scenarios:
2536                  * 1. Stale disk was connected.
2537                  */
2538                 /* Previous state should be NEW. */
2539                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2540                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2541                     g_mirror_disk_state2str(disk->d_state)));
2542                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2543                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2544                     g_mirror_device_state2str(sc->sc_state),
2545                     g_mirror_get_diskname(disk),
2546                     g_mirror_disk_state2str(disk->d_state)));
2547                 /*
2548                  * STALE state is only possible if device is marked
2549                  * NOAUTOSYNC.
2550                  */
2551                 KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2552                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2553                     g_mirror_device_state2str(sc->sc_state),
2554                     g_mirror_get_diskname(disk),
2555                     g_mirror_disk_state2str(disk->d_state)));
2556                 DISK_STATE_CHANGED();
2557
2558                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2559                 disk->d_state = state;
2560                 g_mirror_update_metadata(disk);
2561                 G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2562                     sc->sc_name, g_mirror_get_diskname(disk));
2563                 break;
2564         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2565                 /*
2566                  * Possible scenarios:
2567                  * 1. Disk which needs synchronization was connected.
2568                  */
2569                 /* Previous state should be NEW. */
2570                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2571                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2572                     g_mirror_disk_state2str(disk->d_state)));
2573                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2574                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2575                     g_mirror_device_state2str(sc->sc_state),
2576                     g_mirror_get_diskname(disk),
2577                     g_mirror_disk_state2str(disk->d_state)));
2578                 DISK_STATE_CHANGED();
2579
2580                 if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2581                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2582                 disk->d_state = state;
2583                 if (sc->sc_provider != NULL) {
2584                         g_mirror_sync_start(disk);
2585                         g_mirror_update_metadata(disk);
2586                 }
2587                 break;
2588         case G_MIRROR_DISK_STATE_DISCONNECTED:
2589                 /*
2590                  * Possible scenarios:
2591                  * 1. Device wasn't running yet, but disk disappear.
2592                  * 2. Disk was active and disapppear.
2593                  * 3. Disk disappear during synchronization process.
2594                  */
2595                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2596                         /*
2597                          * Previous state should be ACTIVE, STALE or
2598                          * SYNCHRONIZING.
2599                          */
2600                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2601                             disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2602                             disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2603                             ("Wrong disk state (%s, %s).",
2604                             g_mirror_get_diskname(disk),
2605                             g_mirror_disk_state2str(disk->d_state)));
2606                 } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2607                         /* Previous state should be NEW. */
2608                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2609                             ("Wrong disk state (%s, %s).",
2610                             g_mirror_get_diskname(disk),
2611                             g_mirror_disk_state2str(disk->d_state)));
2612                         /*
2613                          * Reset bumping syncid if disk disappeared in STARTING
2614                          * state.
2615                          */
2616                         if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2617                                 sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2618 #ifdef  INVARIANTS
2619                 } else {
2620                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2621                             sc->sc_name,
2622                             g_mirror_device_state2str(sc->sc_state),
2623                             g_mirror_get_diskname(disk),
2624                             g_mirror_disk_state2str(disk->d_state)));
2625 #endif
2626                 }
2627                 DISK_STATE_CHANGED();
2628                 G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2629                     sc->sc_name, g_mirror_get_diskname(disk));
2630
2631                 g_mirror_destroy_disk(disk);
2632                 break;
2633         case G_MIRROR_DISK_STATE_DESTROY:
2634             {
2635                 int error;
2636
2637                 error = g_mirror_clear_metadata(disk);
2638                 if (error != 0)
2639                         return (error);
2640                 DISK_STATE_CHANGED();
2641                 G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2642                     sc->sc_name, g_mirror_get_diskname(disk));
2643
2644                 g_mirror_destroy_disk(disk);
2645                 sc->sc_ndisks--;
2646                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2647                         g_mirror_update_metadata(disk);
2648                 }
2649                 break;
2650             }
2651         default:
2652                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2653                 break;
2654         }
2655         return (0);
2656 }
2657 #undef  DISK_STATE_CHANGED
2658
2659 int
2660 g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2661 {
2662         struct g_provider *pp;
2663         u_char *buf;
2664         int error;
2665
2666         g_topology_assert();
2667
2668         error = g_access(cp, 1, 0, 0);
2669         if (error != 0)
2670                 return (error);
2671         pp = cp->provider;
2672         g_topology_unlock();
2673         /* Metadata are stored on last sector. */
2674         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2675             &error);
2676         g_topology_lock();
2677         g_access(cp, -1, 0, 0);
2678         if (buf == NULL) {
2679                 G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2680                     cp->provider->name, error);
2681                 return (error);
2682         }
2683
2684         /* Decode metadata. */
2685         error = mirror_metadata_decode(buf, md);
2686         g_free(buf);
2687         if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2688                 return (EINVAL);
2689         if (md->md_version > G_MIRROR_VERSION) {
2690                 G_MIRROR_DEBUG(0,
2691                     "Kernel module is too old to handle metadata from %s.",
2692                     cp->provider->name);
2693                 return (EINVAL);
2694         }
2695         if (error != 0) {
2696                 G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2697                     cp->provider->name);
2698                 return (error);
2699         }
2700
2701         return (0);
2702 }
2703
2704 static int
2705 g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2706     struct g_mirror_metadata *md)
2707 {
2708
2709         if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2710                 G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2711                     pp->name, md->md_did);
2712                 return (EEXIST);
2713         }
2714         if (md->md_all != sc->sc_ndisks) {
2715                 G_MIRROR_DEBUG(1,
2716                     "Invalid '%s' field on disk %s (device %s), skipping.",
2717                     "md_all", pp->name, sc->sc_name);
2718                 return (EINVAL);
2719         }
2720         if (md->md_slice != sc->sc_slice) {
2721                 G_MIRROR_DEBUG(1,
2722                     "Invalid '%s' field on disk %s (device %s), skipping.",
2723                     "md_slice", pp->name, sc->sc_name);
2724                 return (EINVAL);
2725         }
2726         if (md->md_balance != sc->sc_balance) {
2727                 G_MIRROR_DEBUG(1,
2728                     "Invalid '%s' field on disk %s (device %s), skipping.",
2729                     "md_balance", pp->name, sc->sc_name);
2730                 return (EINVAL);
2731         }
2732 #if 0
2733         if (md->md_mediasize != sc->sc_mediasize) {
2734                 G_MIRROR_DEBUG(1,
2735                     "Invalid '%s' field on disk %s (device %s), skipping.",
2736                     "md_mediasize", pp->name, sc->sc_name);
2737                 return (EINVAL);
2738         }
2739 #endif
2740         if (sc->sc_mediasize > pp->mediasize) {
2741                 G_MIRROR_DEBUG(1,
2742                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2743                     sc->sc_name);
2744                 return (EINVAL);
2745         }
2746         if (md->md_sectorsize != sc->sc_sectorsize) {
2747                 G_MIRROR_DEBUG(1,
2748                     "Invalid '%s' field on disk %s (device %s), skipping.",
2749                     "md_sectorsize", pp->name, sc->sc_name);
2750                 return (EINVAL);
2751         }
2752         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2753                 G_MIRROR_DEBUG(1,
2754                     "Invalid sector size of disk %s (device %s), skipping.",
2755                     pp->name, sc->sc_name);
2756                 return (EINVAL);
2757         }
2758         if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2759                 G_MIRROR_DEBUG(1,
2760                     "Invalid device flags on disk %s (device %s), skipping.",
2761                     pp->name, sc->sc_name);
2762                 return (EINVAL);
2763         }
2764         if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2765                 G_MIRROR_DEBUG(1,
2766                     "Invalid disk flags on disk %s (device %s), skipping.",
2767                     pp->name, sc->sc_name);
2768                 return (EINVAL);
2769         }
2770         return (0);
2771 }
2772
2773 int
2774 g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2775     struct g_mirror_metadata *md)
2776 {
2777         struct g_mirror_disk *disk;
2778         int error;
2779
2780         g_topology_assert_not();
2781         G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2782
2783         error = g_mirror_check_metadata(sc, pp, md);
2784         if (error != 0)
2785                 return (error);
2786         if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
2787             md->md_genid < sc->sc_genid) {
2788                 G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2789                     pp->name, sc->sc_name);
2790                 return (EINVAL);
2791         }
2792         disk = g_mirror_init_disk(sc, pp, md, &error);
2793         if (disk == NULL)
2794                 return (error);
2795         error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2796             G_MIRROR_EVENT_WAIT);
2797         if (error != 0)
2798                 return (error);
2799         if (md->md_version < G_MIRROR_VERSION) {
2800                 G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2801                     pp->name, md->md_version, G_MIRROR_VERSION);
2802                 g_mirror_update_metadata(disk);
2803         }
2804         return (0);
2805 }
2806
2807 static void
2808 g_mirror_destroy_delayed(void *arg, int flag)
2809 {
2810         struct g_mirror_softc *sc;
2811         int error;
2812
2813         if (flag == EV_CANCEL) {
2814                 G_MIRROR_DEBUG(1, "Destroying canceled.");
2815                 return;
2816         }
2817         sc = arg;
2818         g_topology_unlock();
2819         sx_xlock(&sc->sc_lock);
2820         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
2821             ("DESTROY flag set on %s.", sc->sc_name));
2822         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
2823             ("DESTROYING flag not set on %s.", sc->sc_name));
2824         G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
2825         error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
2826         if (error != 0) {
2827                 G_MIRROR_DEBUG(0, "Cannot destroy %s.", sc->sc_name);
2828                 sx_xunlock(&sc->sc_lock);
2829         }
2830         g_topology_lock();
2831 }
2832
2833 static int
2834 g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
2835 {
2836         struct g_mirror_softc *sc;
2837         int dcr, dcw, dce, error = 0;
2838
2839         g_topology_assert();
2840         G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
2841             acw, ace);
2842
2843         sc = pp->geom->softc;
2844         if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
2845                 return (0);
2846         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
2847
2848         dcr = pp->acr + acr;
2849         dcw = pp->acw + acw;
2850         dce = pp->ace + ace;
2851
2852         g_topology_unlock();
2853         sx_xlock(&sc->sc_lock);
2854         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
2855             LIST_EMPTY(&sc->sc_disks)) {
2856                 if (acr > 0 || acw > 0 || ace > 0)
2857                         error = ENXIO;
2858                 goto end;
2859         }
2860         if (dcw == 0)
2861                 g_mirror_idle(sc, dcw);
2862         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
2863                 if (acr > 0 || acw > 0 || ace > 0) {
2864                         error = ENXIO;
2865                         goto end;
2866                 }
2867                 if (dcr == 0 && dcw == 0 && dce == 0) {
2868                         g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
2869                             sc, NULL);
2870                 }
2871         }
2872 end:
2873         sx_xunlock(&sc->sc_lock);
2874         g_topology_lock();
2875         return (error);
2876 }
2877
2878 static struct g_geom *
2879 g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
2880 {
2881         struct g_mirror_softc *sc;
2882         struct g_geom *gp;
2883         int error, timeout;
2884
2885         g_topology_assert();
2886         G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
2887             md->md_mid);
2888
2889         /* One disk is minimum. */
2890         if (md->md_all < 1)
2891                 return (NULL);
2892         /*
2893          * Action geom.
2894          */
2895         gp = g_new_geomf(mp, "%s", md->md_name);
2896         sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
2897         gp->start = g_mirror_start;
2898         gp->orphan = g_mirror_orphan;
2899         gp->access = g_mirror_access;
2900         gp->dumpconf = g_mirror_dumpconf;
2901
2902         sc->sc_id = md->md_mid;
2903         sc->sc_slice = md->md_slice;
2904         sc->sc_balance = md->md_balance;
2905         sc->sc_mediasize = md->md_mediasize;
2906         sc->sc_sectorsize = md->md_sectorsize;
2907         sc->sc_ndisks = md->md_all;
2908         sc->sc_flags = md->md_mflags;
2909         sc->sc_bump_id = 0;
2910         sc->sc_idle = 1;
2911         sc->sc_last_write = time_uptime;
2912         sc->sc_writes = 0;
2913         sx_init(&sc->sc_lock, "gmirror:lock");
2914         bioq_init(&sc->sc_queue);
2915         mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
2916         bioq_init(&sc->sc_regular_delayed);
2917         bioq_init(&sc->sc_inflight);
2918         bioq_init(&sc->sc_sync_delayed);
2919         LIST_INIT(&sc->sc_disks);
2920         TAILQ_INIT(&sc->sc_events);
2921         mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
2922         callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
2923         mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
2924         sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
2925         gp->softc = sc;
2926         sc->sc_geom = gp;
2927         sc->sc_provider = NULL;
2928         /*
2929          * Synchronization geom.
2930          */
2931         gp = g_new_geomf(mp, "%s.sync", md->md_name);
2932         gp->softc = sc;
2933         gp->orphan = g_mirror_orphan;
2934         sc->sc_sync.ds_geom = gp;
2935         sc->sc_sync.ds_ndisks = 0;
2936         error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
2937             "g_mirror %s", md->md_name);
2938         if (error != 0) {
2939                 G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
2940                     sc->sc_name);
2941                 g_destroy_geom(sc->sc_sync.ds_geom);
2942                 mtx_destroy(&sc->sc_done_mtx);
2943                 mtx_destroy(&sc->sc_events_mtx);
2944                 mtx_destroy(&sc->sc_queue_mtx);
2945                 sx_destroy(&sc->sc_lock);
2946                 g_destroy_geom(sc->sc_geom);
2947                 free(sc, M_MIRROR);
2948                 return (NULL);
2949         }
2950
2951         G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
2952             sc->sc_name, sc->sc_ndisks, sc->sc_id);
2953
2954         sc->sc_rootmount = root_mount_hold("GMIRROR");
2955         G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
2956         /*
2957          * Run timeout.
2958          */
2959         timeout = g_mirror_timeout * hz;
2960         callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
2961         return (sc->sc_geom);
2962 }
2963
2964 int
2965 g_mirror_destroy(struct g_mirror_softc *sc, int how)
2966 {
2967         struct g_mirror_disk *disk;
2968         struct g_provider *pp;
2969
2970         g_topology_assert_not();
2971         if (sc == NULL)
2972                 return (ENXIO);
2973         sx_assert(&sc->sc_lock, SX_XLOCKED);
2974
2975         pp = sc->sc_provider;
2976         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
2977                 switch (how) {
2978                 case G_MIRROR_DESTROY_SOFT:
2979                         G_MIRROR_DEBUG(1,
2980                             "Device %s is still open (r%dw%de%d).", pp->name,
2981                             pp->acr, pp->acw, pp->ace);
2982                         return (EBUSY);
2983                 case G_MIRROR_DESTROY_DELAYED:
2984                         G_MIRROR_DEBUG(1,
2985                             "Device %s will be destroyed on last close.",
2986                             pp->name);
2987                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2988                                 if (disk->d_state ==
2989                                     G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2990                                         g_mirror_sync_stop(disk, 1);
2991                                 }
2992                         }
2993                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
2994                         return (EBUSY);
2995                 case G_MIRROR_DESTROY_HARD:
2996                         G_MIRROR_DEBUG(1, "Device %s is still open, so it "
2997                             "can't be definitely removed.", pp->name);
2998                 }
2999         }
3000
3001         g_topology_lock();
3002         if (sc->sc_geom->softc == NULL) {
3003                 g_topology_unlock();
3004                 return (0);
3005         }
3006         sc->sc_geom->softc = NULL;
3007         sc->sc_sync.ds_geom->softc = NULL;
3008         g_topology_unlock();
3009
3010         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3011         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
3012         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3013         sx_xunlock(&sc->sc_lock);
3014         mtx_lock(&sc->sc_queue_mtx);
3015         wakeup(sc);
3016         mtx_unlock(&sc->sc_queue_mtx);
3017         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3018         while (sc->sc_worker != NULL)
3019                 tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3020         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3021         sx_xlock(&sc->sc_lock);
3022         g_mirror_destroy_device(sc);
3023         free(sc, M_MIRROR);
3024         return (0);
3025 }
3026
3027 static void
3028 g_mirror_taste_orphan(struct g_consumer *cp)
3029 {
3030
3031         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3032             cp->provider->name));
3033 }
3034
3035 static struct g_geom *
3036 g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3037 {
3038         struct g_mirror_metadata md;
3039         struct g_mirror_softc *sc;
3040         struct g_consumer *cp;
3041         struct g_geom *gp;
3042         int error;
3043
3044         g_topology_assert();
3045         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3046         G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3047
3048         gp = g_new_geomf(mp, "mirror:taste");
3049         /*
3050          * This orphan function should be never called.
3051          */
3052         gp->orphan = g_mirror_taste_orphan;
3053         cp = g_new_consumer(gp);
3054         g_attach(cp, pp);
3055         error = g_mirror_read_metadata(cp, &md);
3056         g_detach(cp);
3057         g_destroy_consumer(cp);
3058         g_destroy_geom(gp);
3059         if (error != 0)
3060                 return (NULL);
3061         gp = NULL;
3062
3063         if (md.md_provider[0] != '\0' &&
3064             !g_compare_names(md.md_provider, pp->name))
3065                 return (NULL);
3066         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3067                 return (NULL);
3068         if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3069                 G_MIRROR_DEBUG(0,
3070                     "Device %s: provider %s marked as inactive, skipping.",
3071                     md.md_name, pp->name);
3072                 return (NULL);
3073         }
3074         if (g_mirror_debug >= 2)
3075                 mirror_metadata_dump(&md);
3076
3077         /*
3078          * Let's check if device already exists.
3079          */
3080         sc = NULL;
3081         LIST_FOREACH(gp, &mp->geom, geom) {
3082                 sc = gp->softc;
3083                 if (sc == NULL)
3084                         continue;
3085                 if (sc->sc_sync.ds_geom == gp)
3086                         continue;
3087                 if (strcmp(md.md_name, sc->sc_name) != 0)
3088                         continue;
3089                 if (md.md_mid != sc->sc_id) {
3090                         G_MIRROR_DEBUG(0, "Device %s already configured.",
3091                             sc->sc_name);
3092                         return (NULL);
3093                 }
3094                 break;
3095         }
3096         if (gp == NULL) {
3097                 gp = g_mirror_create(mp, &md);
3098                 if (gp == NULL) {
3099                         G_MIRROR_DEBUG(0, "Cannot create device %s.",
3100                             md.md_name);
3101                         return (NULL);
3102                 }
3103                 sc = gp->softc;
3104         }
3105         G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3106         g_topology_unlock();
3107         sx_xlock(&sc->sc_lock);
3108         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3109         error = g_mirror_add_disk(sc, pp, &md);
3110         if (error != 0) {
3111                 G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3112                     pp->name, gp->name, error);
3113                 if (LIST_EMPTY(&sc->sc_disks)) {
3114                         g_cancel_event(sc);
3115                         g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3116                         g_topology_lock();
3117                         return (NULL);
3118                 }
3119                 gp = NULL;
3120         }
3121         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3122         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3123                 g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3124                 g_topology_lock();
3125                 return (NULL);
3126         }
3127         sx_xunlock(&sc->sc_lock);
3128         g_topology_lock();
3129         return (gp);
3130 }
3131
3132 static void
3133 g_mirror_resize(struct g_consumer *cp)
3134 {
3135         struct g_mirror_disk *disk;
3136
3137         g_topology_assert();
3138         g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3139
3140         disk = cp->private;
3141         if (disk == NULL)
3142                 return;
3143         g_topology_unlock();
3144         g_mirror_update_metadata(disk);
3145         g_topology_lock();
3146 }
3147
3148 static int
3149 g_mirror_destroy_geom(struct gctl_req *req __unused,
3150     struct g_class *mp __unused, struct g_geom *gp)
3151 {
3152         struct g_mirror_softc *sc;
3153         int error;
3154
3155         g_topology_unlock();
3156         sc = gp->softc;
3157         sx_xlock(&sc->sc_lock);
3158         g_cancel_event(sc);
3159         error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3160         if (error != 0)
3161                 sx_xunlock(&sc->sc_lock);
3162         g_topology_lock();
3163         return (error);
3164 }
3165
3166 static void
3167 g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3168     struct g_consumer *cp, struct g_provider *pp)
3169 {
3170         struct g_mirror_softc *sc;
3171
3172         g_topology_assert();
3173
3174         sc = gp->softc;
3175         if (sc == NULL)
3176                 return;
3177         /* Skip synchronization geom. */
3178         if (gp == sc->sc_sync.ds_geom)
3179                 return;
3180         if (pp != NULL) {
3181                 /* Nothing here. */
3182         } else if (cp != NULL) {
3183                 struct g_mirror_disk *disk;
3184
3185                 disk = cp->private;
3186                 if (disk == NULL)
3187                         return;
3188                 g_topology_unlock();
3189                 sx_xlock(&sc->sc_lock);
3190                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3191                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3192                         sbuf_printf(sb, "%s<Synchronized>", indent);
3193                         if (disk->d_sync.ds_offset == 0)
3194                                 sbuf_printf(sb, "0%%");
3195                         else {
3196                                 sbuf_printf(sb, "%u%%",
3197                                     (u_int)((disk->d_sync.ds_offset * 100) /
3198                                     sc->sc_provider->mediasize));
3199                         }
3200                         sbuf_printf(sb, "</Synchronized>\n");
3201                         if (disk->d_sync.ds_offset > 0) {
3202                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3203                                     "</BytesSynced>\n", indent,
3204                                     (intmax_t)disk->d_sync.ds_offset);
3205                         }
3206                 }
3207                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3208                     disk->d_sync.ds_syncid);
3209                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3210                     disk->d_genid);
3211                 sbuf_printf(sb, "%s<Flags>", indent);
3212                 if (disk->d_flags == 0)
3213                         sbuf_printf(sb, "NONE");
3214                 else {
3215                         int first = 1;
3216
3217 #define ADD_FLAG(flag, name)    do {                                    \
3218         if ((disk->d_flags & (flag)) != 0) {                            \
3219                 if (!first)                                             \
3220                         sbuf_printf(sb, ", ");                          \
3221                 else                                                    \
3222                         first = 0;                                      \
3223                 sbuf_printf(sb, name);                                  \
3224         }                                                               \
3225 } while (0)
3226                         ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3227                         ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3228                         ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3229                         ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3230                             "SYNCHRONIZING");
3231                         ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3232                         ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3233 #undef  ADD_FLAG
3234                 }
3235                 sbuf_printf(sb, "</Flags>\n");
3236                 sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3237                     disk->d_priority);
3238                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3239                     g_mirror_disk_state2str(disk->d_state));
3240                 sx_xunlock(&sc->sc_lock);
3241                 g_topology_lock();
3242         } else {
3243                 g_topology_unlock();
3244                 sx_xlock(&sc->sc_lock);
3245                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3246                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3247                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3248                 sbuf_printf(sb, "%s<Flags>", indent);
3249                 if (sc->sc_flags == 0)
3250                         sbuf_printf(sb, "NONE");
3251                 else {
3252                         int first = 1;
3253
3254 #define ADD_FLAG(flag, name)    do {                                    \
3255         if ((sc->sc_flags & (flag)) != 0) {                             \
3256                 if (!first)                                             \
3257                         sbuf_printf(sb, ", ");                          \
3258                 else                                                    \
3259                         first = 0;                                      \
3260                 sbuf_printf(sb, name);                                  \
3261         }                                                               \
3262 } while (0)
3263                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3264                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3265 #undef  ADD_FLAG
3266                 }
3267                 sbuf_printf(sb, "</Flags>\n");
3268                 sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3269                     (u_int)sc->sc_slice);
3270                 sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3271                     balance_name(sc->sc_balance));
3272                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3273                     sc->sc_ndisks);
3274                 sbuf_printf(sb, "%s<State>", indent);
3275                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3276                         sbuf_printf(sb, "%s", "STARTING");
3277                 else if (sc->sc_ndisks ==
3278                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3279                         sbuf_printf(sb, "%s", "COMPLETE");
3280                 else
3281                         sbuf_printf(sb, "%s", "DEGRADED");
3282                 sbuf_printf(sb, "</State>\n");
3283                 sx_xunlock(&sc->sc_lock);
3284                 g_topology_lock();
3285         }
3286 }
3287
3288 static void
3289 g_mirror_shutdown_post_sync(void *arg, int howto)
3290 {
3291         struct g_class *mp;
3292         struct g_geom *gp, *gp2;
3293         struct g_mirror_softc *sc;
3294         int error;
3295
3296         mp = arg;
3297         DROP_GIANT();
3298         g_topology_lock();
3299         g_mirror_shutdown = 1;
3300         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3301                 if ((sc = gp->softc) == NULL)
3302                         continue;
3303                 /* Skip synchronization geom. */
3304                 if (gp == sc->sc_sync.ds_geom)
3305                         continue;
3306                 g_topology_unlock();
3307                 sx_xlock(&sc->sc_lock);
3308                 g_mirror_idle(sc, -1);
3309                 g_cancel_event(sc);
3310                 error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3311                 if (error != 0)
3312                         sx_xunlock(&sc->sc_lock);
3313                 g_topology_lock();
3314         }
3315         g_topology_unlock();
3316         PICKUP_GIANT();
3317 }
3318
3319 static void
3320 g_mirror_init(struct g_class *mp)
3321 {
3322
3323         g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3324             g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3325         if (g_mirror_post_sync == NULL)
3326                 G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3327 }
3328
3329 static void
3330 g_mirror_fini(struct g_class *mp)
3331 {
3332
3333         if (g_mirror_post_sync != NULL)
3334                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3335 }
3336
3337 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);