sys/geom/mirror/g_mirror.c

   1 /*-
   2  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/module.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/bio.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/malloc.h>
  41 #include <sys/eventhandler.h>
  42 #include <vm/uma.h>
  43 #include <geom/geom.h>
  44 #include <sys/proc.h>
  45 #include <sys/kthread.h>
  46 #include <sys/sched.h>
  47 #include <geom/mirror/g_mirror.h>
  48
  49 FEATURE(geom_mirror, "GEOM mirroring support");
  50
  51 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
  52
  53 SYSCTL_DECL(_kern_geom);
  54 static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
  55     "GEOM_MIRROR stuff");
  56 u_int g_mirror_debug = 0;
  57 TUNABLE_INT("kern.geom.mirror.debug", &g_mirror_debug);
  58 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RW, &g_mirror_debug, 0,
  59     "Debug level");
  60 static u_int g_mirror_timeout = 4;
  61 TUNABLE_INT("kern.geom.mirror.timeout", &g_mirror_timeout);
  62 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RW, &g_mirror_timeout,
  63     0, "Time to wait on all mirror components");
  64 static u_int g_mirror_idletime = 5;
  65 TUNABLE_INT("kern.geom.mirror.idletime", &g_mirror_idletime);
  66 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RW,
  67     &g_mirror_idletime, 0, "Mark components as clean when idling");
  68 static u_int g_mirror_disconnect_on_failure = 1;
  69 TUNABLE_INT("kern.geom.mirror.disconnect_on_failure",
  70     &g_mirror_disconnect_on_failure);
  71 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RW,
  72     &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  73 static u_int g_mirror_syncreqs = 2;
  74 TUNABLE_INT("kern.geom.mirror.sync_requests", &g_mirror_syncreqs);
  75 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  76     &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
  77
  78 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  79         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));       \
  80         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  81         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));       \
  82 } while (0)
  83
  84 static eventhandler_tag g_mirror_post_sync = NULL;
  85 static int g_mirror_shutdown = 0;
  86
  87 static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
  88     struct g_geom *gp);
  89 static g_taste_t g_mirror_taste;
  90 static g_resize_t g_mirror_resize;
  91 static void g_mirror_init(struct g_class *mp);
  92 static void g_mirror_fini(struct g_class *mp);
  93
  94 struct g_class g_mirror_class = {
  95         .name = G_MIRROR_CLASS_NAME,
  96         .version = G_VERSION,
  97         .ctlreq = g_mirror_config,
  98         .taste = g_mirror_taste,
  99         .destroy_geom = g_mirror_destroy_geom,
 100         .init = g_mirror_init,
 101         .fini = g_mirror_fini,
 102         .resize = g_mirror_resize
 103 };
 104
 105
 106 static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
 107 static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
 108 static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
 109 static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
 110     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 111 static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
 112 static void g_mirror_register_request(struct bio *bp);
 113 static void g_mirror_sync_release(struct g_mirror_softc *sc);
 114
 115
 116 static const char *
 117 g_mirror_disk_state2str(int state)
 118 {
 119
 120         switch (state) {
 121         case G_MIRROR_DISK_STATE_NONE:
 122                 return ("NONE");
 123         case G_MIRROR_DISK_STATE_NEW:
 124                 return ("NEW");
 125         case G_MIRROR_DISK_STATE_ACTIVE:
 126                 return ("ACTIVE");
 127         case G_MIRROR_DISK_STATE_STALE:
 128                 return ("STALE");
 129         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 130                 return ("SYNCHRONIZING");
 131         case G_MIRROR_DISK_STATE_DISCONNECTED:
 132                 return ("DISCONNECTED");
 133         case G_MIRROR_DISK_STATE_DESTROY:
 134                 return ("DESTROY");
 135         default:
 136                 return ("INVALID");
 137         }
 138 }
 139
 140 static const char *
 141 g_mirror_device_state2str(int state)
 142 {
 143
 144         switch (state) {
 145         case G_MIRROR_DEVICE_STATE_STARTING:
 146                 return ("STARTING");
 147         case G_MIRROR_DEVICE_STATE_RUNNING:
 148                 return ("RUNNING");
 149         default:
 150                 return ("INVALID");
 151         }
 152 }
 153
 154 static const char *
 155 g_mirror_get_diskname(struct g_mirror_disk *disk)
 156 {
 157
 158         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 159                 return ("[unknown]");
 160         return (disk->d_name);
 161 }
 162
 163 /*
 164  * --- Events handling functions ---
 165  * Events in geom_mirror are used to maintain disks and device status
 166  * from one thread to simplify locking.
 167  */
 168 static void
 169 g_mirror_event_free(struct g_mirror_event *ep)
 170 {
 171
 172         free(ep, M_MIRROR);
 173 }
 174
 175 int
 176 g_mirror_event_send(void *arg, int state, int flags)
 177 {
 178         struct g_mirror_softc *sc;
 179         struct g_mirror_disk *disk;
 180         struct g_mirror_event *ep;
 181         int error;
 182
 183         ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
 184         G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 185         if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
 186                 disk = NULL;
 187                 sc = arg;
 188         } else {
 189                 disk = arg;
 190                 sc = disk->d_softc;
 191         }
 192         ep->e_disk = disk;
 193         ep->e_state = state;
 194         ep->e_flags = flags;
 195         ep->e_error = 0;
 196         mtx_lock(&sc->sc_events_mtx);
 197         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 198         mtx_unlock(&sc->sc_events_mtx);
 199         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 200         mtx_lock(&sc->sc_queue_mtx);
 201         wakeup(sc);
 202         mtx_unlock(&sc->sc_queue_mtx);
 203         if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 204                 return (0);
 205         sx_assert(&sc->sc_lock, SX_XLOCKED);
 206         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 207         sx_xunlock(&sc->sc_lock);
 208         while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
 209                 mtx_lock(&sc->sc_events_mtx);
 210                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
 211                     hz * 5);
 212         }
 213         error = ep->e_error;
 214         g_mirror_event_free(ep);
 215         sx_xlock(&sc->sc_lock);
 216         return (error);
 217 }
 218
 219 static struct g_mirror_event *
 220 g_mirror_event_get(struct g_mirror_softc *sc)
 221 {
 222         struct g_mirror_event *ep;
 223
 224         mtx_lock(&sc->sc_events_mtx);
 225         ep = TAILQ_FIRST(&sc->sc_events);
 226         mtx_unlock(&sc->sc_events_mtx);
 227         return (ep);
 228 }
 229
 230 static void
 231 g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
 232 {
 233
 234         mtx_lock(&sc->sc_events_mtx);
 235         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 236         mtx_unlock(&sc->sc_events_mtx);
 237 }
 238
 239 static void
 240 g_mirror_event_cancel(struct g_mirror_disk *disk)
 241 {
 242         struct g_mirror_softc *sc;
 243         struct g_mirror_event *ep, *tmpep;
 244
 245         sc = disk->d_softc;
 246         sx_assert(&sc->sc_lock, SX_XLOCKED);
 247
 248         mtx_lock(&sc->sc_events_mtx);
 249         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 250                 if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
 251                         continue;
 252                 if (ep->e_disk != disk)
 253                         continue;
 254                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 255                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 256                         g_mirror_event_free(ep);
 257                 else {
 258                         ep->e_error = ECANCELED;
 259                         wakeup(ep);
 260                 }
 261         }
 262         mtx_unlock(&sc->sc_events_mtx);
 263 }
 264
 265 /*
 266  * Return the number of disks in given state.
 267  * If state is equal to -1, count all connected disks.
 268  */
 269 u_int
 270 g_mirror_ndisks(struct g_mirror_softc *sc, int state)
 271 {
 272         struct g_mirror_disk *disk;
 273         u_int n = 0;
 274
 275         sx_assert(&sc->sc_lock, SX_LOCKED);
 276
 277         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 278                 if (state == -1 || disk->d_state == state)
 279                         n++;
 280         }
 281         return (n);
 282 }
 283
 284 /*
 285  * Find a disk in mirror by its disk ID.
 286  */
 287 static struct g_mirror_disk *
 288 g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
 289 {
 290         struct g_mirror_disk *disk;
 291
 292         sx_assert(&sc->sc_lock, SX_XLOCKED);
 293
 294         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 295                 if (disk->d_id == id)
 296                         return (disk);
 297         }
 298         return (NULL);
 299 }
 300
 301 static u_int
 302 g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
 303 {
 304         struct bio *bp;
 305         u_int nreqs = 0;
 306
 307         mtx_lock(&sc->sc_queue_mtx);
 308         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 309                 if (bp->bio_from == cp)
 310                         nreqs++;
 311         }
 312         mtx_unlock(&sc->sc_queue_mtx);
 313         return (nreqs);
 314 }
 315
 316 static int
 317 g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
 318 {
 319
 320         if (cp->index > 0) {
 321                 G_MIRROR_DEBUG(2,
 322                     "I/O requests for %s exist, can't destroy it now.",
 323                     cp->provider->name);
 324                 return (1);
 325         }
 326         if (g_mirror_nrequests(sc, cp) > 0) {
 327                 G_MIRROR_DEBUG(2,
 328                     "I/O requests for %s in queue, can't destroy it now.",
 329                     cp->provider->name);
 330                 return (1);
 331         }
 332         return (0);
 333 }
 334
 335 static void
 336 g_mirror_destroy_consumer(void *arg, int flags __unused)
 337 {
 338         struct g_consumer *cp;
 339
 340         g_topology_assert();
 341
 342         cp = arg;
 343         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 344         g_detach(cp);
 345         g_destroy_consumer(cp);
 346 }
 347
 348 static void
 349 g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 350 {
 351         struct g_provider *pp;
 352         int retaste_wait;
 353
 354         g_topology_assert();
 355
 356         cp->private = NULL;
 357         if (g_mirror_is_busy(sc, cp))
 358                 return;
 359         pp = cp->provider;
 360         retaste_wait = 0;
 361         if (cp->acw == 1) {
 362                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 363                         retaste_wait = 1;
 364         }
 365         G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 366             -cp->acw, -cp->ace, 0);
 367         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 368                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 369         if (retaste_wait) {
 370                 /*
 371                  * After retaste event was send (inside g_access()), we can send
 372                  * event to detach and destroy consumer.
 373                  * A class, which has consumer to the given provider connected
 374                  * will not receive retaste event for the provider.
 375                  * This is the way how I ignore retaste events when I close
 376                  * consumers opened for write: I detach and destroy consumer
 377                  * after retaste event is sent.
 378                  */
 379                 g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
 380                 return;
 381         }
 382         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
 383         g_detach(cp);
 384         g_destroy_consumer(cp);
 385 }
 386
 387 static int
 388 g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
 389 {
 390         struct g_consumer *cp;
 391         int error;
 392
 393         g_topology_assert_not();
 394         KASSERT(disk->d_consumer == NULL,
 395             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 396
 397         g_topology_lock();
 398         cp = g_new_consumer(disk->d_softc->sc_geom);
 399         cp->flags |= G_CF_DIRECT_RECEIVE;
 400         error = g_attach(cp, pp);
 401         if (error != 0) {
 402                 g_destroy_consumer(cp);
 403                 g_topology_unlock();
 404                 return (error);
 405         }
 406         error = g_access(cp, 1, 1, 1);
 407         if (error != 0) {
 408                 g_detach(cp);
 409                 g_destroy_consumer(cp);
 410                 g_topology_unlock();
 411                 G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
 412                     pp->name, error);
 413                 return (error);
 414         }
 415         g_topology_unlock();
 416         disk->d_consumer = cp;
 417         disk->d_consumer->private = disk;
 418         disk->d_consumer->index = 0;
 419
 420         G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
 421         return (0);
 422 }
 423
 424 static void
 425 g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 426 {
 427
 428         g_topology_assert();
 429
 430         if (cp == NULL)
 431                 return;
 432         if (cp->provider != NULL)
 433                 g_mirror_kill_consumer(sc, cp);
 434         else
 435                 g_destroy_consumer(cp);
 436 }
 437
 438 /*
 439  * Initialize disk. This means allocate memory, create consumer, attach it
 440  * to the provider and open access (r1w1e1) to it.
 441  */
 442 static struct g_mirror_disk *
 443 g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 444     struct g_mirror_metadata *md, int *errorp)
 445 {
 446         struct g_mirror_disk *disk;
 447         int i, error;
 448
 449         disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
 450         if (disk == NULL) {
 451                 error = ENOMEM;
 452                 goto fail;
 453         }
 454         disk->d_softc = sc;
 455         error = g_mirror_connect_disk(disk, pp);
 456         if (error != 0)
 457                 goto fail;
 458         disk->d_id = md->md_did;
 459         disk->d_state = G_MIRROR_DISK_STATE_NONE;
 460         disk->d_priority = md->md_priority;
 461         disk->d_flags = md->md_dflags;
 462         error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
 463         if (error == 0 && i != 0)
 464                 disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 465         if (md->md_provider[0] != '\0')
 466                 disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
 467         disk->d_sync.ds_consumer = NULL;
 468         disk->d_sync.ds_offset = md->md_sync_offset;
 469         disk->d_sync.ds_offset_done = md->md_sync_offset;
 470         disk->d_genid = md->md_genid;
 471         disk->d_sync.ds_syncid = md->md_syncid;
 472         if (errorp != NULL)
 473                 *errorp = 0;
 474         return (disk);
 475 fail:
 476         if (errorp != NULL)
 477                 *errorp = error;
 478         if (disk != NULL)
 479                 free(disk, M_MIRROR);
 480         return (NULL);
 481 }
 482
 483 static void
 484 g_mirror_destroy_disk(struct g_mirror_disk *disk)
 485 {
 486         struct g_mirror_softc *sc;
 487
 488         g_topology_assert_not();
 489         sc = disk->d_softc;
 490         sx_assert(&sc->sc_lock, SX_XLOCKED);
 491
 492         LIST_REMOVE(disk, d_next);
 493         g_mirror_event_cancel(disk);
 494         if (sc->sc_hint == disk)
 495                 sc->sc_hint = NULL;
 496         switch (disk->d_state) {
 497         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 498                 g_mirror_sync_stop(disk, 1);
 499                 /* FALLTHROUGH */
 500         case G_MIRROR_DISK_STATE_NEW:
 501         case G_MIRROR_DISK_STATE_STALE:
 502         case G_MIRROR_DISK_STATE_ACTIVE:
 503                 g_topology_lock();
 504                 g_mirror_disconnect_consumer(sc, disk->d_consumer);
 505                 g_topology_unlock();
 506                 free(disk, M_MIRROR);
 507                 break;
 508         default:
 509                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 510                     g_mirror_get_diskname(disk),
 511                     g_mirror_disk_state2str(disk->d_state)));
 512         }
 513 }
 514
 515 static void
 516 g_mirror_destroy_device(struct g_mirror_softc *sc)
 517 {
 518         struct g_mirror_disk *disk;
 519         struct g_mirror_event *ep;
 520         struct g_geom *gp;
 521         struct g_consumer *cp, *tmpcp;
 522
 523         g_topology_assert_not();
 524         sx_assert(&sc->sc_lock, SX_XLOCKED);
 525
 526         gp = sc->sc_geom;
 527         if (sc->sc_provider != NULL)
 528                 g_mirror_destroy_provider(sc);
 529         for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
 530             disk = LIST_FIRST(&sc->sc_disks)) {
 531                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 532                 g_mirror_update_metadata(disk);
 533                 g_mirror_destroy_disk(disk);
 534         }
 535         while ((ep = g_mirror_event_get(sc)) != NULL) {
 536                 g_mirror_event_remove(sc, ep);
 537                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 538                         g_mirror_event_free(ep);
 539                 else {
 540                         ep->e_error = ECANCELED;
 541                         ep->e_flags |= G_MIRROR_EVENT_DONE;
 542                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 543                         mtx_lock(&sc->sc_events_mtx);
 544                         wakeup(ep);
 545                         mtx_unlock(&sc->sc_events_mtx);
 546                 }
 547         }
 548         callout_drain(&sc->sc_callout);
 549
 550         g_topology_lock();
 551         LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
 552                 g_mirror_disconnect_consumer(sc, cp);
 553         }
 554         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 555         G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
 556         g_wither_geom(gp, ENXIO);
 557         g_topology_unlock();
 558         mtx_destroy(&sc->sc_queue_mtx);
 559         mtx_destroy(&sc->sc_events_mtx);
 560         mtx_destroy(&sc->sc_done_mtx);
 561         sx_xunlock(&sc->sc_lock);
 562         sx_destroy(&sc->sc_lock);
 563 }
 564
 565 static void
 566 g_mirror_orphan(struct g_consumer *cp)
 567 {
 568         struct g_mirror_disk *disk;
 569
 570         g_topology_assert();
 571
 572         disk = cp->private;
 573         if (disk == NULL)
 574                 return;
 575         disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 576         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 577             G_MIRROR_EVENT_DONTWAIT);
 578 }
 579
 580 /*
 581  * Function should return the next active disk on the list.
 582  * It is possible that it will be the same disk as given.
 583  * If there are no active disks on list, NULL is returned.
 584  */
 585 static __inline struct g_mirror_disk *
 586 g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
 587 {
 588         struct g_mirror_disk *dp;
 589
 590         for (dp = LIST_NEXT(disk, d_next); dp != disk;
 591             dp = LIST_NEXT(dp, d_next)) {
 592                 if (dp == NULL)
 593                         dp = LIST_FIRST(&sc->sc_disks);
 594                 if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
 595                         break;
 596         }
 597         if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 598                 return (NULL);
 599         return (dp);
 600 }
 601
 602 static struct g_mirror_disk *
 603 g_mirror_get_disk(struct g_mirror_softc *sc)
 604 {
 605         struct g_mirror_disk *disk;
 606
 607         if (sc->sc_hint == NULL) {
 608                 sc->sc_hint = LIST_FIRST(&sc->sc_disks);
 609                 if (sc->sc_hint == NULL)
 610                         return (NULL);
 611         }
 612         disk = sc->sc_hint;
 613         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
 614                 disk = g_mirror_find_next(sc, disk);
 615                 if (disk == NULL)
 616                         return (NULL);
 617         }
 618         sc->sc_hint = g_mirror_find_next(sc, disk);
 619         return (disk);
 620 }
 621
 622 static int
 623 g_mirror_write_metadata(struct g_mirror_disk *disk,
 624     struct g_mirror_metadata *md)
 625 {
 626         struct g_mirror_softc *sc;
 627         struct g_consumer *cp;
 628         off_t offset, length;
 629         u_char *sector;
 630         int error = 0;
 631
 632         g_topology_assert_not();
 633         sc = disk->d_softc;
 634         sx_assert(&sc->sc_lock, SX_LOCKED);
 635
 636         cp = disk->d_consumer;
 637         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 638         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 639         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 640             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 641             cp->acw, cp->ace));
 642         length = cp->provider->sectorsize;
 643         offset = cp->provider->mediasize - length;
 644         sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
 645         if (md != NULL &&
 646             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
 647                 /*
 648                  * Handle the case, when the size of parent provider reduced.
 649                  */
 650                 if (offset < md->md_mediasize)
 651                         error = ENOSPC;
 652                 else
 653                         mirror_metadata_encode(md, sector);
 654         }
 655         if (error == 0)
 656                 error = g_write_data(cp, offset, sector, length);
 657         free(sector, M_MIRROR);
 658         if (error != 0) {
 659                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 660                         disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 661                         G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
 662                             "(device=%s, error=%d).",
 663                             g_mirror_get_diskname(disk), sc->sc_name, error);
 664                 } else {
 665                         G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
 666                             "(device=%s, error=%d).",
 667                             g_mirror_get_diskname(disk), sc->sc_name, error);
 668                 }
 669                 if (g_mirror_disconnect_on_failure &&
 670                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 671                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 672                         g_mirror_event_send(disk,
 673                             G_MIRROR_DISK_STATE_DISCONNECTED,
 674                             G_MIRROR_EVENT_DONTWAIT);
 675                 }
 676         }
 677         return (error);
 678 }
 679
 680 static int
 681 g_mirror_clear_metadata(struct g_mirror_disk *disk)
 682 {
 683         int error;
 684
 685         g_topology_assert_not();
 686         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 687
 688         error = g_mirror_write_metadata(disk, NULL);
 689         if (error == 0) {
 690                 G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
 691                     g_mirror_get_diskname(disk));
 692         } else {
 693                 G_MIRROR_DEBUG(0,
 694                     "Cannot clear metadata on disk %s (error=%d).",
 695                     g_mirror_get_diskname(disk), error);
 696         }
 697         return (error);
 698 }
 699
 700 void
 701 g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
 702     struct g_mirror_metadata *md)
 703 {
 704
 705         strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
 706         md->md_version = G_MIRROR_VERSION;
 707         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 708         md->md_mid = sc->sc_id;
 709         md->md_all = sc->sc_ndisks;
 710         md->md_slice = sc->sc_slice;
 711         md->md_balance = sc->sc_balance;
 712         md->md_genid = sc->sc_genid;
 713         md->md_mediasize = sc->sc_mediasize;
 714         md->md_sectorsize = sc->sc_sectorsize;
 715         md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
 716         bzero(md->md_provider, sizeof(md->md_provider));
 717         if (disk == NULL) {
 718                 md->md_did = arc4random();
 719                 md->md_priority = 0;
 720                 md->md_syncid = 0;
 721                 md->md_dflags = 0;
 722                 md->md_sync_offset = 0;
 723                 md->md_provsize = 0;
 724         } else {
 725                 md->md_did = disk->d_id;
 726                 md->md_priority = disk->d_priority;
 727                 md->md_syncid = disk->d_sync.ds_syncid;
 728                 md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
 729                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
 730                         md->md_sync_offset = disk->d_sync.ds_offset_done;
 731                 else
 732                         md->md_sync_offset = 0;
 733                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
 734                         strlcpy(md->md_provider,
 735                             disk->d_consumer->provider->name,
 736                             sizeof(md->md_provider));
 737                 }
 738                 md->md_provsize = disk->d_consumer->provider->mediasize;
 739         }
 740 }
 741
 742 void
 743 g_mirror_update_metadata(struct g_mirror_disk *disk)
 744 {
 745         struct g_mirror_softc *sc;
 746         struct g_mirror_metadata md;
 747         int error;
 748
 749         g_topology_assert_not();
 750         sc = disk->d_softc;
 751         sx_assert(&sc->sc_lock, SX_LOCKED);
 752
 753         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
 754                 g_mirror_fill_metadata(sc, disk, &md);
 755         error = g_mirror_write_metadata(disk, &md);
 756         if (error == 0) {
 757                 G_MIRROR_DEBUG(2, "Metadata on %s updated.",
 758                     g_mirror_get_diskname(disk));
 759         } else {
 760                 G_MIRROR_DEBUG(0,
 761                     "Cannot update metadata on disk %s (error=%d).",
 762                     g_mirror_get_diskname(disk), error);
 763         }
 764 }
 765
 766 static void
 767 g_mirror_bump_syncid(struct g_mirror_softc *sc)
 768 {
 769         struct g_mirror_disk *disk;
 770
 771         g_topology_assert_not();
 772         sx_assert(&sc->sc_lock, SX_XLOCKED);
 773         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 774             ("%s called with no active disks (device=%s).", __func__,
 775             sc->sc_name));
 776
 777         sc->sc_syncid++;
 778         G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 779             sc->sc_syncid);
 780         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 781                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 782                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 783                         disk->d_sync.ds_syncid = sc->sc_syncid;
 784                         g_mirror_update_metadata(disk);
 785                 }
 786         }
 787 }
 788
 789 static void
 790 g_mirror_bump_genid(struct g_mirror_softc *sc)
 791 {
 792         struct g_mirror_disk *disk;
 793
 794         g_topology_assert_not();
 795         sx_assert(&sc->sc_lock, SX_XLOCKED);
 796         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 797             ("%s called with no active disks (device=%s).", __func__,
 798             sc->sc_name));
 799
 800         sc->sc_genid++;
 801         G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 802             sc->sc_genid);
 803         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 804                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 805                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 806                         disk->d_genid = sc->sc_genid;
 807                         g_mirror_update_metadata(disk);
 808                 }
 809         }
 810 }
 811
 812 static int
 813 g_mirror_idle(struct g_mirror_softc *sc, int acw)
 814 {
 815         struct g_mirror_disk *disk;
 816         int timeout;
 817
 818         g_topology_assert_not();
 819         sx_assert(&sc->sc_lock, SX_XLOCKED);
 820
 821         if (sc->sc_provider == NULL)
 822                 return (0);
 823         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 824                 return (0);
 825         if (sc->sc_idle)
 826                 return (0);
 827         if (sc->sc_writes > 0)
 828                 return (0);
 829         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 830                 timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
 831                 if (!g_mirror_shutdown && timeout > 0)
 832                         return (timeout);
 833         }
 834         sc->sc_idle = 1;
 835         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 836                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 837                         continue;
 838                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
 839                     g_mirror_get_diskname(disk), sc->sc_name);
 840                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 841                 g_mirror_update_metadata(disk);
 842         }
 843         return (0);
 844 }
 845
 846 static void
 847 g_mirror_unidle(struct g_mirror_softc *sc)
 848 {
 849         struct g_mirror_disk *disk;
 850
 851         g_topology_assert_not();
 852         sx_assert(&sc->sc_lock, SX_XLOCKED);
 853
 854         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 855                 return;
 856         sc->sc_idle = 0;
 857         sc->sc_last_write = time_uptime;
 858         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 859                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 860                         continue;
 861                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
 862                     g_mirror_get_diskname(disk), sc->sc_name);
 863                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
 864                 g_mirror_update_metadata(disk);
 865         }
 866 }
 867
 868 static void
 869 g_mirror_flush_done(struct bio *bp)
 870 {
 871         struct g_mirror_softc *sc;
 872         struct bio *pbp;
 873
 874         pbp = bp->bio_parent;
 875         sc = pbp->bio_to->geom->softc;
 876         mtx_lock(&sc->sc_done_mtx);
 877         if (pbp->bio_error == 0)
 878                 pbp->bio_error = bp->bio_error;
 879         pbp->bio_completed += bp->bio_completed;
 880         pbp->bio_inbed++;
 881         if (pbp->bio_children == pbp->bio_inbed) {
 882                 mtx_unlock(&sc->sc_done_mtx);
 883                 g_io_deliver(pbp, pbp->bio_error);
 884         } else
 885                 mtx_unlock(&sc->sc_done_mtx);
 886         g_destroy_bio(bp);
 887 }
 888
 889 static void
 890 g_mirror_done(struct bio *bp)
 891 {
 892         struct g_mirror_softc *sc;
 893
 894         sc = bp->bio_from->geom->softc;
 895         bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
 896         mtx_lock(&sc->sc_queue_mtx);
 897         bioq_insert_tail(&sc->sc_queue, bp);
 898         mtx_unlock(&sc->sc_queue_mtx);
 899         wakeup(sc);
 900 }
 901
 902 static void
 903 g_mirror_regular_request(struct bio *bp)
 904 {
 905         struct g_mirror_softc *sc;
 906         struct g_mirror_disk *disk;
 907         struct bio *pbp;
 908
 909         g_topology_assert_not();
 910
 911         pbp = bp->bio_parent;
 912         sc = pbp->bio_to->geom->softc;
 913         bp->bio_from->index--;
 914         if (bp->bio_cmd == BIO_WRITE)
 915                 sc->sc_writes--;
 916         disk = bp->bio_from->private;
 917         if (disk == NULL) {
 918                 g_topology_lock();
 919                 g_mirror_kill_consumer(sc, bp->bio_from);
 920                 g_topology_unlock();
 921         }
 922
 923         pbp->bio_inbed++;
 924         KASSERT(pbp->bio_inbed <= pbp->bio_children,
 925             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
 926             pbp->bio_children));
 927         if (bp->bio_error == 0 && pbp->bio_error == 0) {
 928                 G_MIRROR_LOGREQ(3, bp, "Request delivered.");
 929                 g_destroy_bio(bp);
 930                 if (pbp->bio_children == pbp->bio_inbed) {
 931                         G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
 932                         pbp->bio_completed = pbp->bio_length;
 933                         if (pbp->bio_cmd == BIO_WRITE ||
 934                             pbp->bio_cmd == BIO_DELETE) {
 935                                 bioq_remove(&sc->sc_inflight, pbp);
 936                                 /* Release delayed sync requests if possible. */
 937                                 g_mirror_sync_release(sc);
 938                         }
 939                         g_io_deliver(pbp, pbp->bio_error);
 940                 }
 941                 return;
 942         } else if (bp->bio_error != 0) {
 943                 if (pbp->bio_error == 0)
 944                         pbp->bio_error = bp->bio_error;
 945                 if (disk != NULL) {
 946                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 947                                 disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 948                                 G_MIRROR_LOGREQ(0, bp,
 949                                     "Request failed (error=%d).",
 950                                     bp->bio_error);
 951                         } else {
 952                                 G_MIRROR_LOGREQ(1, bp,
 953                                     "Request failed (error=%d).",
 954                                     bp->bio_error);
 955                         }
 956                         if (g_mirror_disconnect_on_failure &&
 957                             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
 958                         {
 959                                 sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 960                                 g_mirror_event_send(disk,
 961                                     G_MIRROR_DISK_STATE_DISCONNECTED,
 962                                     G_MIRROR_EVENT_DONTWAIT);
 963                         }
 964                 }
 965                 switch (pbp->bio_cmd) {
 966                 case BIO_DELETE:
 967                 case BIO_WRITE:
 968                         pbp->bio_inbed--;
 969                         pbp->bio_children--;
 970                         break;
 971                 }
 972         }
 973         g_destroy_bio(bp);
 974
 975         switch (pbp->bio_cmd) {
 976         case BIO_READ:
 977                 if (pbp->bio_inbed < pbp->bio_children)
 978                         break;
 979                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
 980                         g_io_deliver(pbp, pbp->bio_error);
 981                 else {
 982                         pbp->bio_error = 0;
 983                         mtx_lock(&sc->sc_queue_mtx);
 984                         bioq_insert_tail(&sc->sc_queue, pbp);
 985                         mtx_unlock(&sc->sc_queue_mtx);
 986                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 987                         wakeup(sc);
 988                 }
 989                 break;
 990         case BIO_DELETE:
 991         case BIO_WRITE:
 992                 if (pbp->bio_children == 0) {
 993                         /*
 994                          * All requests failed.
 995                          */
 996                 } else if (pbp->bio_inbed < pbp->bio_children) {
 997                         /* Do nothing. */
 998                         break;
 999                 } else if (pbp->bio_children == pbp->bio_inbed) {
1000                         /* Some requests succeeded. */
1001                         pbp->bio_error = 0;
1002                         pbp->bio_completed = pbp->bio_length;
1003                 }
1004                 bioq_remove(&sc->sc_inflight, pbp);
1005                 /* Release delayed sync requests if possible. */
1006                 g_mirror_sync_release(sc);
1007                 g_io_deliver(pbp, pbp->bio_error);
1008                 break;
1009         default:
1010                 KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1011                 break;
1012         }
1013 }
1014
1015 static void
1016 g_mirror_sync_done(struct bio *bp)
1017 {
1018         struct g_mirror_softc *sc;
1019
1020         G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1021         sc = bp->bio_from->geom->softc;
1022         bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1023         mtx_lock(&sc->sc_queue_mtx);
1024         bioq_insert_tail(&sc->sc_queue, bp);
1025         mtx_unlock(&sc->sc_queue_mtx);
1026         wakeup(sc);
1027 }
1028
1029 static void
1030 g_mirror_candelete(struct bio *bp)
1031 {
1032         struct g_mirror_softc *sc;
1033         struct g_mirror_disk *disk;
1034         int *val;
1035
1036         sc = bp->bio_to->geom->softc;
1037         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1038                 if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
1039                         break;
1040         }
1041         val = (int *)bp->bio_data;
1042         *val = (disk != NULL);
1043         g_io_deliver(bp, 0);
1044 }
1045
1046 static void
1047 g_mirror_kernel_dump(struct bio *bp)
1048 {
1049         struct g_mirror_softc *sc;
1050         struct g_mirror_disk *disk;
1051         struct bio *cbp;
1052         struct g_kerneldump *gkd;
1053
1054         /*
1055          * We configure dumping to the first component, because this component
1056          * will be used for reading with 'prefer' balance algorithm.
1057          * If the component with the higest priority is currently disconnected
1058          * we will not be able to read the dump after the reboot if it will be
1059          * connected and synchronized later. Can we do something better?
1060          */
1061         sc = bp->bio_to->geom->softc;
1062         disk = LIST_FIRST(&sc->sc_disks);
1063
1064         gkd = (struct g_kerneldump *)bp->bio_data;
1065         if (gkd->length > bp->bio_to->mediasize)
1066                 gkd->length = bp->bio_to->mediasize;
1067         cbp = g_clone_bio(bp);
1068         if (cbp == NULL) {
1069                 g_io_deliver(bp, ENOMEM);
1070                 return;
1071         }
1072         cbp->bio_done = g_std_done;
1073         g_io_request(cbp, disk->d_consumer);
1074         G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1075             g_mirror_get_diskname(disk));
1076 }
1077
1078 static void
1079 g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
1080 {
1081         struct bio_queue_head queue;
1082         struct g_mirror_disk *disk;
1083         struct g_consumer *cp;
1084         struct bio *cbp;
1085
1086         bioq_init(&queue);
1087         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1088                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1089                         continue;
1090                 cbp = g_clone_bio(bp);
1091                 if (cbp == NULL) {
1092                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1093                                 g_destroy_bio(cbp);
1094                         if (bp->bio_error == 0)
1095                                 bp->bio_error = ENOMEM;
1096                         g_io_deliver(bp, bp->bio_error);
1097                         return;
1098                 }
1099                 bioq_insert_tail(&queue, cbp);
1100                 cbp->bio_done = g_mirror_flush_done;
1101                 cbp->bio_caller1 = disk;
1102                 cbp->bio_to = disk->d_consumer->provider;
1103         }
1104         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1105                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1106                 disk = cbp->bio_caller1;
1107                 cbp->bio_caller1 = NULL;
1108                 cp = disk->d_consumer;
1109                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1110                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1111                     cp->acr, cp->acw, cp->ace));
1112                 g_io_request(cbp, disk->d_consumer);
1113         }
1114 }
1115
1116 static void
1117 g_mirror_start(struct bio *bp)
1118 {
1119         struct g_mirror_softc *sc;
1120
1121         sc = bp->bio_to->geom->softc;
1122         /*
1123          * If sc == NULL or there are no valid disks, provider's error
1124          * should be set and g_mirror_start() should not be called at all.
1125          */
1126         KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1127             ("Provider's error should be set (error=%d)(mirror=%s).",
1128             bp->bio_to->error, bp->bio_to->name));
1129         G_MIRROR_LOGREQ(3, bp, "Request received.");
1130
1131         switch (bp->bio_cmd) {
1132         case BIO_READ:
1133         case BIO_WRITE:
1134         case BIO_DELETE:
1135                 break;
1136         case BIO_FLUSH:
1137                 g_mirror_flush(sc, bp);
1138                 return;
1139         case BIO_GETATTR:
1140                 if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
1141                         g_mirror_candelete(bp);
1142                         return;
1143                 } else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1144                         g_mirror_kernel_dump(bp);
1145                         return;
1146                 }
1147                 /* FALLTHROUGH */
1148         default:
1149                 g_io_deliver(bp, EOPNOTSUPP);
1150                 return;
1151         }
1152         mtx_lock(&sc->sc_queue_mtx);
1153         bioq_insert_tail(&sc->sc_queue, bp);
1154         mtx_unlock(&sc->sc_queue_mtx);
1155         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1156         wakeup(sc);
1157 }
1158
1159 /*
1160  * Return TRUE if the given request is colliding with a in-progress
1161  * synchronization request.
1162  */
1163 static int
1164 g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1165 {
1166         struct g_mirror_disk *disk;
1167         struct bio *sbp;
1168         off_t rstart, rend, sstart, send;
1169         u_int i;
1170
1171         if (sc->sc_sync.ds_ndisks == 0)
1172                 return (0);
1173         rstart = bp->bio_offset;
1174         rend = bp->bio_offset + bp->bio_length;
1175         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1176                 if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1177                         continue;
1178                 for (i = 0; i < g_mirror_syncreqs; i++) {
1179                         sbp = disk->d_sync.ds_bios[i];
1180                         if (sbp == NULL)
1181                                 continue;
1182                         sstart = sbp->bio_offset;
1183                         send = sbp->bio_offset + sbp->bio_length;
1184                         if (rend > sstart && rstart < send)
1185                                 return (1);
1186                 }
1187         }
1188         return (0);
1189 }
1190
1191 /*
1192  * Return TRUE if the given sync request is colliding with a in-progress regular
1193  * request.
1194  */
1195 static int
1196 g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1197 {
1198         off_t rstart, rend, sstart, send;
1199         struct bio *bp;
1200
1201         if (sc->sc_sync.ds_ndisks == 0)
1202                 return (0);
1203         sstart = sbp->bio_offset;
1204         send = sbp->bio_offset + sbp->bio_length;
1205         TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1206                 rstart = bp->bio_offset;
1207                 rend = bp->bio_offset + bp->bio_length;
1208                 if (rend > sstart && rstart < send)
1209                         return (1);
1210         }
1211         return (0);
1212 }
1213
1214 /*
1215  * Puts request onto delayed queue.
1216  */
1217 static void
1218 g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1219 {
1220
1221         G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1222         bioq_insert_head(&sc->sc_regular_delayed, bp);
1223 }
1224
1225 /*
1226  * Puts synchronization request onto delayed queue.
1227  */
1228 static void
1229 g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1230 {
1231
1232         G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1233         bioq_insert_tail(&sc->sc_sync_delayed, bp);
1234 }
1235
1236 /*
1237  * Releases delayed regular requests which don't collide anymore with sync
1238  * requests.
1239  */
1240 static void
1241 g_mirror_regular_release(struct g_mirror_softc *sc)
1242 {
1243         struct bio *bp, *bp2;
1244
1245         TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1246                 if (g_mirror_sync_collision(sc, bp))
1247                         continue;
1248                 bioq_remove(&sc->sc_regular_delayed, bp);
1249                 G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1250                 mtx_lock(&sc->sc_queue_mtx);
1251                 bioq_insert_head(&sc->sc_queue, bp);
1252 #if 0
1253                 /*
1254                  * wakeup() is not needed, because this function is called from
1255                  * the worker thread.
1256                  */
1257                 wakeup(&sc->sc_queue);
1258 #endif
1259                 mtx_unlock(&sc->sc_queue_mtx);
1260         }
1261 }
1262
1263 /*
1264  * Releases delayed sync requests which don't collide anymore with regular
1265  * requests.
1266  */
1267 static void
1268 g_mirror_sync_release(struct g_mirror_softc *sc)
1269 {
1270         struct bio *bp, *bp2;
1271
1272         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1273                 if (g_mirror_regular_collision(sc, bp))
1274                         continue;
1275                 bioq_remove(&sc->sc_sync_delayed, bp);
1276                 G_MIRROR_LOGREQ(2, bp,
1277                     "Releasing delayed synchronization request.");
1278                 g_io_request(bp, bp->bio_from);
1279         }
1280 }
1281
1282 /*
1283  * Handle synchronization requests.
1284  * Every synchronization request is two-steps process: first, READ request is
1285  * send to active provider and then WRITE request (with read data) to the provider
1286  * beeing synchronized. When WRITE is finished, new synchronization request is
1287  * send.
1288  */
1289 static void
1290 g_mirror_sync_request(struct bio *bp)
1291 {
1292         struct g_mirror_softc *sc;
1293         struct g_mirror_disk *disk;
1294
1295         bp->bio_from->index--;
1296         sc = bp->bio_from->geom->softc;
1297         disk = bp->bio_from->private;
1298         if (disk == NULL) {
1299                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1300                 g_topology_lock();
1301                 g_mirror_kill_consumer(sc, bp->bio_from);
1302                 g_topology_unlock();
1303                 free(bp->bio_data, M_MIRROR);
1304                 g_destroy_bio(bp);
1305                 sx_xlock(&sc->sc_lock);
1306                 return;
1307         }
1308
1309         /*
1310          * Synchronization request.
1311          */
1312         switch (bp->bio_cmd) {
1313         case BIO_READ:
1314             {
1315                 struct g_consumer *cp;
1316
1317                 if (bp->bio_error != 0) {
1318                         G_MIRROR_LOGREQ(0, bp,
1319                             "Synchronization request failed (error=%d).",
1320                             bp->bio_error);
1321                         g_destroy_bio(bp);
1322                         return;
1323                 }
1324                 G_MIRROR_LOGREQ(3, bp,
1325                     "Synchronization request half-finished.");
1326                 bp->bio_cmd = BIO_WRITE;
1327                 bp->bio_cflags = 0;
1328                 cp = disk->d_consumer;
1329                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1330                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1331                     cp->acr, cp->acw, cp->ace));
1332                 cp->index++;
1333                 g_io_request(bp, cp);
1334                 return;
1335             }
1336         case BIO_WRITE:
1337             {
1338                 struct g_mirror_disk_sync *sync;
1339                 off_t offset;
1340                 void *data;
1341                 int i;
1342
1343                 if (bp->bio_error != 0) {
1344                         G_MIRROR_LOGREQ(0, bp,
1345                             "Synchronization request failed (error=%d).",
1346                             bp->bio_error);
1347                         g_destroy_bio(bp);
1348                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1349                         g_mirror_event_send(disk,
1350                             G_MIRROR_DISK_STATE_DISCONNECTED,
1351                             G_MIRROR_EVENT_DONTWAIT);
1352                         return;
1353                 }
1354                 G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1355                 sync = &disk->d_sync;
1356                 if (sync->ds_offset >= sc->sc_mediasize ||
1357                     sync->ds_consumer == NULL ||
1358                     (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1359                         /* Don't send more synchronization requests. */
1360                         sync->ds_inflight--;
1361                         if (sync->ds_bios != NULL) {
1362                                 i = (int)(uintptr_t)bp->bio_caller1;
1363                                 sync->ds_bios[i] = NULL;
1364                         }
1365                         free(bp->bio_data, M_MIRROR);
1366                         g_destroy_bio(bp);
1367                         if (sync->ds_inflight > 0)
1368                                 return;
1369                         if (sync->ds_consumer == NULL ||
1370                             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1371                                 return;
1372                         }
1373                         /* Disk up-to-date, activate it. */
1374                         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1375                             G_MIRROR_EVENT_DONTWAIT);
1376                         return;
1377                 }
1378
1379                 /* Send next synchronization request. */
1380                 data = bp->bio_data;
1381                 bzero(bp, sizeof(*bp));
1382                 bp->bio_cmd = BIO_READ;
1383                 bp->bio_offset = sync->ds_offset;
1384                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1385                 sync->ds_offset += bp->bio_length;
1386                 bp->bio_done = g_mirror_sync_done;
1387                 bp->bio_data = data;
1388                 bp->bio_from = sync->ds_consumer;
1389                 bp->bio_to = sc->sc_provider;
1390                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1391                 sync->ds_consumer->index++;
1392                 /*
1393                  * Delay the request if it is colliding with a regular request.
1394                  */
1395                 if (g_mirror_regular_collision(sc, bp))
1396                         g_mirror_sync_delay(sc, bp);
1397                 else
1398                         g_io_request(bp, sync->ds_consumer);
1399
1400                 /* Release delayed requests if possible. */
1401                 g_mirror_regular_release(sc);
1402
1403                 /* Find the smallest offset */
1404                 offset = sc->sc_mediasize;
1405                 for (i = 0; i < g_mirror_syncreqs; i++) {
1406                         bp = sync->ds_bios[i];
1407                         if (bp->bio_offset < offset)
1408                                 offset = bp->bio_offset;
1409                 }
1410                 if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
1411                         /* Update offset_done on every 100 blocks. */
1412                         sync->ds_offset_done = offset;
1413                         g_mirror_update_metadata(disk);
1414                 }
1415                 return;
1416             }
1417         default:
1418                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1419                     bp->bio_cmd, sc->sc_name));
1420                 break;
1421         }
1422 }
1423
1424 static void
1425 g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1426 {
1427         struct g_mirror_disk *disk;
1428         struct g_consumer *cp;
1429         struct bio *cbp;
1430
1431         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1432                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1433                         break;
1434         }
1435         if (disk == NULL) {
1436                 if (bp->bio_error == 0)
1437                         bp->bio_error = ENXIO;
1438                 g_io_deliver(bp, bp->bio_error);
1439                 return;
1440         }
1441         cbp = g_clone_bio(bp);
1442         if (cbp == NULL) {
1443                 if (bp->bio_error == 0)
1444                         bp->bio_error = ENOMEM;
1445                 g_io_deliver(bp, bp->bio_error);
1446                 return;
1447         }
1448         /*
1449          * Fill in the component buf structure.
1450          */
1451         cp = disk->d_consumer;
1452         cbp->bio_done = g_mirror_done;
1453         cbp->bio_to = cp->provider;
1454         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1455         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1456             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1457             cp->acw, cp->ace));
1458         cp->index++;
1459         g_io_request(cbp, cp);
1460 }
1461
1462 static void
1463 g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1464 {
1465         struct g_mirror_disk *disk;
1466         struct g_consumer *cp;
1467         struct bio *cbp;
1468
1469         disk = g_mirror_get_disk(sc);
1470         if (disk == NULL) {
1471                 if (bp->bio_error == 0)
1472                         bp->bio_error = ENXIO;
1473                 g_io_deliver(bp, bp->bio_error);
1474                 return;
1475         }
1476         cbp = g_clone_bio(bp);
1477         if (cbp == NULL) {
1478                 if (bp->bio_error == 0)
1479                         bp->bio_error = ENOMEM;
1480                 g_io_deliver(bp, bp->bio_error);
1481                 return;
1482         }
1483         /*
1484          * Fill in the component buf structure.
1485          */
1486         cp = disk->d_consumer;
1487         cbp->bio_done = g_mirror_done;
1488         cbp->bio_to = cp->provider;
1489         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1490         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1491             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1492             cp->acw, cp->ace));
1493         cp->index++;
1494         g_io_request(cbp, cp);
1495 }
1496
1497 #define TRACK_SIZE  (1 * 1024 * 1024)
1498 #define LOAD_SCALE      256
1499 #define ABS(x)          (((x) >= 0) ? (x) : (-(x)))
1500
1501 static void
1502 g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1503 {
1504         struct g_mirror_disk *disk, *dp;
1505         struct g_consumer *cp;
1506         struct bio *cbp;
1507         int prio, best;
1508
1509         /* Find a disk with the smallest load. */
1510         disk = NULL;
1511         best = INT_MAX;
1512         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1513                 if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1514                         continue;
1515                 prio = dp->load;
1516                 /* If disk head is precisely in position - highly prefer it. */
1517                 if (dp->d_last_offset == bp->bio_offset)
1518                         prio -= 2 * LOAD_SCALE;
1519                 else
1520                 /* If disk head is close to position - prefer it. */
1521                 if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1522                         prio -= 1 * LOAD_SCALE;
1523                 if (prio <= best) {
1524                         disk = dp;
1525                         best = prio;
1526                 }
1527         }
1528         KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1529         cbp = g_clone_bio(bp);
1530         if (cbp == NULL) {
1531                 if (bp->bio_error == 0)
1532                         bp->bio_error = ENOMEM;
1533                 g_io_deliver(bp, bp->bio_error);
1534                 return;
1535         }
1536         /*
1537          * Fill in the component buf structure.
1538          */
1539         cp = disk->d_consumer;
1540         cbp->bio_done = g_mirror_done;
1541         cbp->bio_to = cp->provider;
1542         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1543         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1544             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1545             cp->acw, cp->ace));
1546         cp->index++;
1547         /* Remember last head position */
1548         disk->d_last_offset = bp->bio_offset + bp->bio_length;
1549         /* Update loads. */
1550         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1551                 dp->load = (dp->d_consumer->index * LOAD_SCALE +
1552                     dp->load * 7) / 8;
1553         }
1554         g_io_request(cbp, cp);
1555 }
1556
1557 static void
1558 g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1559 {
1560         struct bio_queue_head queue;
1561         struct g_mirror_disk *disk;
1562         struct g_consumer *cp;
1563         struct bio *cbp;
1564         off_t left, mod, offset, slice;
1565         u_char *data;
1566         u_int ndisks;
1567
1568         if (bp->bio_length <= sc->sc_slice) {
1569                 g_mirror_request_round_robin(sc, bp);
1570                 return;
1571         }
1572         ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1573         slice = bp->bio_length / ndisks;
1574         mod = slice % sc->sc_provider->sectorsize;
1575         if (mod != 0)
1576                 slice += sc->sc_provider->sectorsize - mod;
1577         /*
1578          * Allocate all bios before sending any request, so we can
1579          * return ENOMEM in nice and clean way.
1580          */
1581         left = bp->bio_length;
1582         offset = bp->bio_offset;
1583         data = bp->bio_data;
1584         bioq_init(&queue);
1585         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1586                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1587                         continue;
1588                 cbp = g_clone_bio(bp);
1589                 if (cbp == NULL) {
1590                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1591                                 g_destroy_bio(cbp);
1592                         if (bp->bio_error == 0)
1593                                 bp->bio_error = ENOMEM;
1594                         g_io_deliver(bp, bp->bio_error);
1595                         return;
1596                 }
1597                 bioq_insert_tail(&queue, cbp);
1598                 cbp->bio_done = g_mirror_done;
1599                 cbp->bio_caller1 = disk;
1600                 cbp->bio_to = disk->d_consumer->provider;
1601                 cbp->bio_offset = offset;
1602                 cbp->bio_data = data;
1603                 cbp->bio_length = MIN(left, slice);
1604                 left -= cbp->bio_length;
1605                 if (left == 0)
1606                         break;
1607                 offset += cbp->bio_length;
1608                 data += cbp->bio_length;
1609         }
1610         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1611                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1612                 disk = cbp->bio_caller1;
1613                 cbp->bio_caller1 = NULL;
1614                 cp = disk->d_consumer;
1615                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1616                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1617                     cp->acr, cp->acw, cp->ace));
1618                 disk->d_consumer->index++;
1619                 g_io_request(cbp, disk->d_consumer);
1620         }
1621 }
1622
1623 static void
1624 g_mirror_register_request(struct bio *bp)
1625 {
1626         struct g_mirror_softc *sc;
1627
1628         sc = bp->bio_to->geom->softc;
1629         switch (bp->bio_cmd) {
1630         case BIO_READ:
1631                 switch (sc->sc_balance) {
1632                 case G_MIRROR_BALANCE_LOAD:
1633                         g_mirror_request_load(sc, bp);
1634                         break;
1635                 case G_MIRROR_BALANCE_PREFER:
1636                         g_mirror_request_prefer(sc, bp);
1637                         break;
1638                 case G_MIRROR_BALANCE_ROUND_ROBIN:
1639                         g_mirror_request_round_robin(sc, bp);
1640                         break;
1641                 case G_MIRROR_BALANCE_SPLIT:
1642                         g_mirror_request_split(sc, bp);
1643                         break;
1644                 }
1645                 return;
1646         case BIO_WRITE:
1647         case BIO_DELETE:
1648             {
1649                 struct g_mirror_disk *disk;
1650                 struct g_mirror_disk_sync *sync;
1651                 struct bio_queue_head queue;
1652                 struct g_consumer *cp;
1653                 struct bio *cbp;
1654
1655                 /*
1656                  * Delay the request if it is colliding with a synchronization
1657                  * request.
1658                  */
1659                 if (g_mirror_sync_collision(sc, bp)) {
1660                         g_mirror_regular_delay(sc, bp);
1661                         return;
1662                 }
1663
1664                 if (sc->sc_idle)
1665                         g_mirror_unidle(sc);
1666                 else
1667                         sc->sc_last_write = time_uptime;
1668
1669                 /*
1670                  * Allocate all bios before sending any request, so we can
1671                  * return ENOMEM in nice and clean way.
1672                  */
1673                 bioq_init(&queue);
1674                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1675                         sync = &disk->d_sync;
1676                         switch (disk->d_state) {
1677                         case G_MIRROR_DISK_STATE_ACTIVE:
1678                                 break;
1679                         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1680                                 if (bp->bio_offset >= sync->ds_offset)
1681                                         continue;
1682                                 break;
1683                         default:
1684                                 continue;
1685                         }
1686                         if (bp->bio_cmd == BIO_DELETE &&
1687                             (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1688                                 continue;
1689                         cbp = g_clone_bio(bp);
1690                         if (cbp == NULL) {
1691                                 while ((cbp = bioq_takefirst(&queue)) != NULL)
1692                                         g_destroy_bio(cbp);
1693                                 if (bp->bio_error == 0)
1694                                         bp->bio_error = ENOMEM;
1695                                 g_io_deliver(bp, bp->bio_error);
1696                                 return;
1697                         }
1698                         bioq_insert_tail(&queue, cbp);
1699                         cbp->bio_done = g_mirror_done;
1700                         cp = disk->d_consumer;
1701                         cbp->bio_caller1 = cp;
1702                         cbp->bio_to = cp->provider;
1703                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1704                             ("Consumer %s not opened (r%dw%de%d).",
1705                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1706                 }
1707                 if (bioq_first(&queue) == NULL) {
1708                         g_io_deliver(bp, EOPNOTSUPP);
1709                         return;
1710                 }
1711                 while ((cbp = bioq_takefirst(&queue)) != NULL) {
1712                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1713                         cp = cbp->bio_caller1;
1714                         cbp->bio_caller1 = NULL;
1715                         cp->index++;
1716                         sc->sc_writes++;
1717                         g_io_request(cbp, cp);
1718                 }
1719                 /*
1720                  * Put request onto inflight queue, so we can check if new
1721                  * synchronization requests don't collide with it.
1722                  */
1723                 bioq_insert_tail(&sc->sc_inflight, bp);
1724                 /*
1725                  * Bump syncid on first write.
1726                  */
1727                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1728                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1729                         g_mirror_bump_syncid(sc);
1730                 }
1731                 return;
1732             }
1733         default:
1734                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1735                     bp->bio_cmd, sc->sc_name));
1736                 break;
1737         }
1738 }
1739
1740 static int
1741 g_mirror_can_destroy(struct g_mirror_softc *sc)
1742 {
1743         struct g_geom *gp;
1744         struct g_consumer *cp;
1745
1746         g_topology_assert();
1747         gp = sc->sc_geom;
1748         if (gp->softc == NULL)
1749                 return (1);
1750         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1751                 return (0);
1752         LIST_FOREACH(cp, &gp->consumer, consumer) {
1753                 if (g_mirror_is_busy(sc, cp))
1754                         return (0);
1755         }
1756         gp = sc->sc_sync.ds_geom;
1757         LIST_FOREACH(cp, &gp->consumer, consumer) {
1758                 if (g_mirror_is_busy(sc, cp))
1759                         return (0);
1760         }
1761         G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1762             sc->sc_name);
1763         return (1);
1764 }
1765
1766 static int
1767 g_mirror_try_destroy(struct g_mirror_softc *sc)
1768 {
1769
1770         if (sc->sc_rootmount != NULL) {
1771                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1772                     sc->sc_rootmount);
1773                 root_mount_rel(sc->sc_rootmount);
1774                 sc->sc_rootmount = NULL;
1775         }
1776         g_topology_lock();
1777         if (!g_mirror_can_destroy(sc)) {
1778                 g_topology_unlock();
1779                 return (0);
1780         }
1781         sc->sc_geom->softc = NULL;
1782         sc->sc_sync.ds_geom->softc = NULL;
1783         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
1784                 g_topology_unlock();
1785                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1786                     &sc->sc_worker);
1787                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
1788                 sx_xunlock(&sc->sc_lock);
1789                 wakeup(&sc->sc_worker);
1790                 sc->sc_worker = NULL;
1791         } else {
1792                 g_topology_unlock();
1793                 g_mirror_destroy_device(sc);
1794                 free(sc, M_MIRROR);
1795         }
1796         return (1);
1797 }
1798
1799 /*
1800  * Worker thread.
1801  */
1802 static void
1803 g_mirror_worker(void *arg)
1804 {
1805         struct g_mirror_softc *sc;
1806         struct g_mirror_event *ep;
1807         struct bio *bp;
1808         int timeout;
1809
1810         sc = arg;
1811         thread_lock(curthread);
1812         sched_prio(curthread, PRIBIO);
1813         thread_unlock(curthread);
1814
1815         sx_xlock(&sc->sc_lock);
1816         for (;;) {
1817                 G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1818                 /*
1819                  * First take a look at events.
1820                  * This is important to handle events before any I/O requests.
1821                  */
1822                 ep = g_mirror_event_get(sc);
1823                 if (ep != NULL) {
1824                         g_mirror_event_remove(sc, ep);
1825                         if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1826                                 /* Update only device status. */
1827                                 G_MIRROR_DEBUG(3,
1828                                     "Running event for device %s.",
1829                                     sc->sc_name);
1830                                 ep->e_error = 0;
1831                                 g_mirror_update_device(sc, 1);
1832                         } else {
1833                                 /* Update disk status. */
1834                                 G_MIRROR_DEBUG(3, "Running event for disk %s.",
1835                                      g_mirror_get_diskname(ep->e_disk));
1836                                 ep->e_error = g_mirror_update_disk(ep->e_disk,
1837                                     ep->e_state);
1838                                 if (ep->e_error == 0)
1839                                         g_mirror_update_device(sc, 0);
1840                         }
1841                         if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1842                                 KASSERT(ep->e_error == 0,
1843                                     ("Error cannot be handled."));
1844                                 g_mirror_event_free(ep);
1845                         } else {
1846                                 ep->e_flags |= G_MIRROR_EVENT_DONE;
1847                                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1848                                     ep);
1849                                 mtx_lock(&sc->sc_events_mtx);
1850                                 wakeup(ep);
1851                                 mtx_unlock(&sc->sc_events_mtx);
1852                         }
1853                         if ((sc->sc_flags &
1854                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1855                                 if (g_mirror_try_destroy(sc)) {
1856                                         curthread->td_pflags &= ~TDP_GEOM;
1857                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1858                                         kproc_exit(0);
1859                                 }
1860                         }
1861                         G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1862                         continue;
1863                 }
1864                 /*
1865                  * Check if we can mark array as CLEAN and if we can't take
1866                  * how much seconds should we wait.
1867                  */
1868                 timeout = g_mirror_idle(sc, -1);
1869                 /*
1870                  * Now I/O requests.
1871                  */
1872                 /* Get first request from the queue. */
1873                 mtx_lock(&sc->sc_queue_mtx);
1874                 bp = bioq_takefirst(&sc->sc_queue);
1875                 if (bp == NULL) {
1876                         if ((sc->sc_flags &
1877                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1878                                 mtx_unlock(&sc->sc_queue_mtx);
1879                                 if (g_mirror_try_destroy(sc)) {
1880                                         curthread->td_pflags &= ~TDP_GEOM;
1881                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1882                                         kproc_exit(0);
1883                                 }
1884                                 mtx_lock(&sc->sc_queue_mtx);
1885                         }
1886                         sx_xunlock(&sc->sc_lock);
1887                         /*
1888                          * XXX: We can miss an event here, because an event
1889                          *      can be added without sx-device-lock and without
1890                          *      mtx-queue-lock. Maybe I should just stop using
1891                          *      dedicated mutex for events synchronization and
1892                          *      stick with the queue lock?
1893                          *      The event will hang here until next I/O request
1894                          *      or next event is received.
1895                          */
1896                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1897                             timeout * hz);
1898                         sx_xlock(&sc->sc_lock);
1899                         G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1900                         continue;
1901                 }
1902                 mtx_unlock(&sc->sc_queue_mtx);
1903
1904                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
1905                     (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
1906                         g_mirror_sync_request(bp);      /* READ */
1907                 } else if (bp->bio_to != sc->sc_provider) {
1908                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
1909                                 g_mirror_regular_request(bp);
1910                         else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
1911                                 g_mirror_sync_request(bp);      /* WRITE */
1912                         else {
1913                                 KASSERT(0,
1914                                     ("Invalid request cflags=0x%hhx to=%s.",
1915                                     bp->bio_cflags, bp->bio_to->name));
1916                         }
1917                 } else {
1918                         g_mirror_register_request(bp);
1919                 }
1920                 G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
1921         }
1922 }
1923
1924 static void
1925 g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
1926 {
1927
1928         sx_assert(&sc->sc_lock, SX_LOCKED);
1929
1930         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
1931                 return;
1932         if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
1933                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
1934                     g_mirror_get_diskname(disk), sc->sc_name);
1935                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1936         } else if (sc->sc_idle &&
1937             (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
1938                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
1939                     g_mirror_get_diskname(disk), sc->sc_name);
1940                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
1941         }
1942 }
1943
1944 static void
1945 g_mirror_sync_start(struct g_mirror_disk *disk)
1946 {
1947         struct g_mirror_softc *sc;
1948         struct g_consumer *cp;
1949         struct bio *bp;
1950         int error, i;
1951
1952         g_topology_assert_not();
1953         sc = disk->d_softc;
1954         sx_assert(&sc->sc_lock, SX_LOCKED);
1955
1956         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
1957             ("Disk %s is not marked for synchronization.",
1958             g_mirror_get_diskname(disk)));
1959         KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1960             ("Device not in RUNNING state (%s, %u).", sc->sc_name,
1961             sc->sc_state));
1962
1963         sx_xunlock(&sc->sc_lock);
1964         g_topology_lock();
1965         cp = g_new_consumer(sc->sc_sync.ds_geom);
1966         cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
1967         error = g_attach(cp, sc->sc_provider);
1968         KASSERT(error == 0,
1969             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
1970         error = g_access(cp, 1, 0, 0);
1971         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
1972         g_topology_unlock();
1973         sx_xlock(&sc->sc_lock);
1974
1975         G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
1976             g_mirror_get_diskname(disk));
1977         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
1978                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1979         KASSERT(disk->d_sync.ds_consumer == NULL,
1980             ("Sync consumer already exists (device=%s, disk=%s).",
1981             sc->sc_name, g_mirror_get_diskname(disk)));
1982
1983         disk->d_sync.ds_consumer = cp;
1984         disk->d_sync.ds_consumer->private = disk;
1985         disk->d_sync.ds_consumer->index = 0;
1986
1987         /*
1988          * Allocate memory for synchronization bios and initialize them.
1989          */
1990         disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
1991             M_MIRROR, M_WAITOK);
1992         for (i = 0; i < g_mirror_syncreqs; i++) {
1993                 bp = g_alloc_bio();
1994                 disk->d_sync.ds_bios[i] = bp;
1995                 bp->bio_parent = NULL;
1996                 bp->bio_cmd = BIO_READ;
1997                 bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
1998                 bp->bio_cflags = 0;
1999                 bp->bio_offset = disk->d_sync.ds_offset;
2000                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
2001                 disk->d_sync.ds_offset += bp->bio_length;
2002                 bp->bio_done = g_mirror_sync_done;
2003                 bp->bio_from = disk->d_sync.ds_consumer;
2004                 bp->bio_to = sc->sc_provider;
2005                 bp->bio_caller1 = (void *)(uintptr_t)i;
2006         }
2007
2008         /* Increase the number of disks in SYNCHRONIZING state. */
2009         sc->sc_sync.ds_ndisks++;
2010         /* Set the number of in-flight synchronization requests. */
2011         disk->d_sync.ds_inflight = g_mirror_syncreqs;
2012
2013         /*
2014          * Fire off first synchronization requests.
2015          */
2016         for (i = 0; i < g_mirror_syncreqs; i++) {
2017                 bp = disk->d_sync.ds_bios[i];
2018                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
2019                 disk->d_sync.ds_consumer->index++;
2020                 /*
2021                  * Delay the request if it is colliding with a regular request.
2022                  */
2023                 if (g_mirror_regular_collision(sc, bp))
2024                         g_mirror_sync_delay(sc, bp);
2025                 else
2026                         g_io_request(bp, disk->d_sync.ds_consumer);
2027         }
2028 }
2029
2030 /*
2031  * Stop synchronization process.
2032  * type: 0 - synchronization finished
2033  *       1 - synchronization stopped
2034  */
2035 static void
2036 g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2037 {
2038         struct g_mirror_softc *sc;
2039         struct g_consumer *cp;
2040
2041         g_topology_assert_not();
2042         sc = disk->d_softc;
2043         sx_assert(&sc->sc_lock, SX_LOCKED);
2044
2045         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2046             ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2047             g_mirror_disk_state2str(disk->d_state)));
2048         if (disk->d_sync.ds_consumer == NULL)
2049                 return;
2050
2051         if (type == 0) {
2052                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2053                     sc->sc_name, g_mirror_get_diskname(disk));
2054         } else /* if (type == 1) */ {
2055                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2056                     sc->sc_name, g_mirror_get_diskname(disk));
2057         }
2058         free(disk->d_sync.ds_bios, M_MIRROR);
2059         disk->d_sync.ds_bios = NULL;
2060         cp = disk->d_sync.ds_consumer;
2061         disk->d_sync.ds_consumer = NULL;
2062         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2063         sc->sc_sync.ds_ndisks--;
2064         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2065         g_topology_lock();
2066         g_mirror_kill_consumer(sc, cp);
2067         g_topology_unlock();
2068         sx_xlock(&sc->sc_lock);
2069 }
2070
2071 static void
2072 g_mirror_launch_provider(struct g_mirror_softc *sc)
2073 {
2074         struct g_mirror_disk *disk;
2075         struct g_provider *pp, *dp;
2076
2077         sx_assert(&sc->sc_lock, SX_LOCKED);
2078
2079         g_topology_lock();
2080         pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2081         pp->flags |= G_PF_DIRECT_RECEIVE;
2082         pp->mediasize = sc->sc_mediasize;
2083         pp->sectorsize = sc->sc_sectorsize;
2084         pp->stripesize = 0;
2085         pp->stripeoffset = 0;
2086
2087         /* Splitting of unmapped BIO's could work but isn't implemented now */
2088         if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2089                 pp->flags |= G_PF_ACCEPT_UNMAPPED;
2090
2091         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2092                 if (disk->d_consumer && disk->d_consumer->provider) {
2093                         dp = disk->d_consumer->provider;
2094                         if (dp->stripesize > pp->stripesize) {
2095                                 pp->stripesize = dp->stripesize;
2096                                 pp->stripeoffset = dp->stripeoffset;
2097                         }
2098                         /* A provider underneath us doesn't support unmapped */
2099                         if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2100                                 G_MIRROR_DEBUG(0, "Cancelling unmapped "
2101                                     "because of %s.", dp->name);
2102                                 pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2103                         }
2104                 }
2105         }
2106         sc->sc_provider = pp;
2107         g_error_provider(pp, 0);
2108         g_topology_unlock();
2109         G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2110             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2111         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2112                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2113                         g_mirror_sync_start(disk);
2114         }
2115 }
2116
2117 static void
2118 g_mirror_destroy_provider(struct g_mirror_softc *sc)
2119 {
2120         struct g_mirror_disk *disk;
2121         struct bio *bp;
2122
2123         g_topology_assert_not();
2124         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2125             sc->sc_name));
2126
2127         g_topology_lock();
2128         g_error_provider(sc->sc_provider, ENXIO);
2129         mtx_lock(&sc->sc_queue_mtx);
2130         while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL) {
2131                 /*
2132                  * Abort any pending I/O that wasn't generated by us.
2133                  * Synchronization requests and requests destined for individual
2134                  * mirror components can be destroyed immediately.
2135                  */
2136                 if (bp->bio_to == sc->sc_provider &&
2137                     bp->bio_from->geom != sc->sc_sync.ds_geom) {
2138                         g_io_deliver(bp, ENXIO);
2139                 } else {
2140                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
2141                                 free(bp->bio_data, M_MIRROR);
2142                         g_destroy_bio(bp);
2143                 }
2144         }
2145         mtx_unlock(&sc->sc_queue_mtx);
2146         G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2147             sc->sc_provider->name);
2148         sc->sc_provider->flags |= G_PF_WITHER;
2149         g_orphan_provider(sc->sc_provider, ENXIO);
2150         g_topology_unlock();
2151         sc->sc_provider = NULL;
2152         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2153                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2154                         g_mirror_sync_stop(disk, 1);
2155         }
2156 }
2157
2158 static void
2159 g_mirror_go(void *arg)
2160 {
2161         struct g_mirror_softc *sc;
2162
2163         sc = arg;
2164         G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2165         g_mirror_event_send(sc, 0,
2166             G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2167 }
2168
2169 static u_int
2170 g_mirror_determine_state(struct g_mirror_disk *disk)
2171 {
2172         struct g_mirror_softc *sc;
2173         u_int state;
2174
2175         sc = disk->d_softc;
2176         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2177                 if ((disk->d_flags &
2178                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2179                         /* Disk does not need synchronization. */
2180                         state = G_MIRROR_DISK_STATE_ACTIVE;
2181                 } else {
2182                         if ((sc->sc_flags &
2183                              G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2184                             (disk->d_flags &
2185                              G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2186                                 /*
2187                                  * We can start synchronization from
2188                                  * the stored offset.
2189                                  */
2190                                 state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2191                         } else {
2192                                 state = G_MIRROR_DISK_STATE_STALE;
2193                         }
2194                 }
2195         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2196                 /*
2197                  * Reset all synchronization data for this disk,
2198                  * because if it even was synchronized, it was
2199                  * synchronized to disks with different syncid.
2200                  */
2201                 disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2202                 disk->d_sync.ds_offset = 0;
2203                 disk->d_sync.ds_offset_done = 0;
2204                 disk->d_sync.ds_syncid = sc->sc_syncid;
2205                 if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2206                     (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2207                         state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2208                 } else {
2209                         state = G_MIRROR_DISK_STATE_STALE;
2210                 }
2211         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2212                 /*
2213                  * Not good, NOT GOOD!
2214                  * It means that mirror was started on stale disks
2215                  * and more fresh disk just arrive.
2216                  * If there were writes, mirror is broken, sorry.
2217                  * I think the best choice here is don't touch
2218                  * this disk and inform the user loudly.
2219                  */
2220                 G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2221                     "disk (%s) arrives!! It will not be connected to the "
2222                     "running device.", sc->sc_name,
2223                     g_mirror_get_diskname(disk));
2224                 g_mirror_destroy_disk(disk);
2225                 state = G_MIRROR_DISK_STATE_NONE;
2226                 /* Return immediately, because disk was destroyed. */
2227                 return (state);
2228         }
2229         G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2230             g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2231         return (state);
2232 }
2233
2234 /*
2235  * Update device state.
2236  */
2237 static void
2238 g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
2239 {
2240         struct g_mirror_disk *disk;
2241         u_int state;
2242
2243         sx_assert(&sc->sc_lock, SX_XLOCKED);
2244
2245         switch (sc->sc_state) {
2246         case G_MIRROR_DEVICE_STATE_STARTING:
2247             {
2248                 struct g_mirror_disk *pdisk, *tdisk;
2249                 u_int dirty, ndisks, genid, syncid;
2250
2251                 KASSERT(sc->sc_provider == NULL,
2252                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2253                 /*
2254                  * Are we ready? We are, if all disks are connected or
2255                  * if we have any disks and 'force' is true.
2256                  */
2257                 ndisks = g_mirror_ndisks(sc, -1);
2258                 if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
2259                         ;
2260                 } else if (ndisks == 0) {
2261                         /*
2262                          * Disks went down in starting phase, so destroy
2263                          * device.
2264                          */
2265                         callout_drain(&sc->sc_callout);
2266                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2267                         G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2268                             sc->sc_rootmount);
2269                         root_mount_rel(sc->sc_rootmount);
2270                         sc->sc_rootmount = NULL;
2271                         return;
2272                 } else {
2273                         return;
2274                 }
2275
2276                 /*
2277                  * Activate all disks with the biggest syncid.
2278                  */
2279                 if (force) {
2280                         /*
2281                          * If 'force' is true, we have been called due to
2282                          * timeout, so don't bother canceling timeout.
2283                          */
2284                         ndisks = 0;
2285                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2286                                 if ((disk->d_flags &
2287                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2288                                         ndisks++;
2289                                 }
2290                         }
2291                         if (ndisks == 0) {
2292                                 /* No valid disks found, destroy device. */
2293                                 sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2294                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2295                                     __LINE__, sc->sc_rootmount);
2296                                 root_mount_rel(sc->sc_rootmount);
2297                                 sc->sc_rootmount = NULL;
2298                                 return;
2299                         }
2300                 } else {
2301                         /* Cancel timeout. */
2302                         callout_drain(&sc->sc_callout);
2303                 }
2304
2305                 /*
2306                  * Find the biggest genid.
2307                  */
2308                 genid = 0;
2309                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2310                         if (disk->d_genid > genid)
2311                                 genid = disk->d_genid;
2312                 }
2313                 sc->sc_genid = genid;
2314                 /*
2315                  * Remove all disks without the biggest genid.
2316                  */
2317                 LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2318                         if (disk->d_genid < genid) {
2319                                 G_MIRROR_DEBUG(0,
2320                                     "Component %s (device %s) broken, skipping.",
2321                                     g_mirror_get_diskname(disk), sc->sc_name);
2322                                 g_mirror_destroy_disk(disk);
2323                         }
2324                 }
2325
2326                 /*
2327                  * Find the biggest syncid.
2328                  */
2329                 syncid = 0;
2330                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2331                         if (disk->d_sync.ds_syncid > syncid)
2332                                 syncid = disk->d_sync.ds_syncid;
2333                 }
2334
2335                 /*
2336                  * Here we need to look for dirty disks and if all disks
2337                  * with the biggest syncid are dirty, we have to choose
2338                  * one with the biggest priority and rebuild the rest.
2339                  */
2340                 /*
2341                  * Find the number of dirty disks with the biggest syncid.
2342                  * Find the number of disks with the biggest syncid.
2343                  * While here, find a disk with the biggest priority.
2344                  */
2345                 dirty = ndisks = 0;
2346                 pdisk = NULL;
2347                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2348                         if (disk->d_sync.ds_syncid != syncid)
2349                                 continue;
2350                         if ((disk->d_flags &
2351                             G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2352                                 continue;
2353                         }
2354                         ndisks++;
2355                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2356                                 dirty++;
2357                                 if (pdisk == NULL ||
2358                                     pdisk->d_priority < disk->d_priority) {
2359                                         pdisk = disk;
2360                                 }
2361                         }
2362                 }
2363                 if (dirty == 0) {
2364                         /* No dirty disks at all, great. */
2365                 } else if (dirty == ndisks) {
2366                         /*
2367                          * Force synchronization for all dirty disks except one
2368                          * with the biggest priority.
2369                          */
2370                         KASSERT(pdisk != NULL, ("pdisk == NULL"));
2371                         G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2372                             "master disk for synchronization.",
2373                             g_mirror_get_diskname(pdisk), sc->sc_name);
2374                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2375                                 if (disk->d_sync.ds_syncid != syncid)
2376                                         continue;
2377                                 if ((disk->d_flags &
2378                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2379                                         continue;
2380                                 }
2381                                 KASSERT((disk->d_flags &
2382                                     G_MIRROR_DISK_FLAG_DIRTY) != 0,
2383                                     ("Disk %s isn't marked as dirty.",
2384                                     g_mirror_get_diskname(disk)));
2385                                 /* Skip the disk with the biggest priority. */
2386                                 if (disk == pdisk)
2387                                         continue;
2388                                 disk->d_sync.ds_syncid = 0;
2389                         }
2390                 } else if (dirty < ndisks) {
2391                         /*
2392                          * Force synchronization for all dirty disks.
2393                          * We have some non-dirty disks.
2394                          */
2395                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2396                                 if (disk->d_sync.ds_syncid != syncid)
2397                                         continue;
2398                                 if ((disk->d_flags &
2399                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2400                                         continue;
2401                                 }
2402                                 if ((disk->d_flags &
2403                                     G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2404                                         continue;
2405                                 }
2406                                 disk->d_sync.ds_syncid = 0;
2407                         }
2408                 }
2409
2410                 /* Reset hint. */
2411                 sc->sc_hint = NULL;
2412                 sc->sc_syncid = syncid;
2413                 if (force) {
2414                         /* Remember to bump syncid on first write. */
2415                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2416                 }
2417                 state = G_MIRROR_DEVICE_STATE_RUNNING;
2418                 G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2419                     sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2420                     g_mirror_device_state2str(state));
2421                 sc->sc_state = state;
2422                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2423                         state = g_mirror_determine_state(disk);
2424                         g_mirror_event_send(disk, state,
2425                             G_MIRROR_EVENT_DONTWAIT);
2426                         if (state == G_MIRROR_DISK_STATE_STALE)
2427                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2428                 }
2429                 break;
2430             }
2431         case G_MIRROR_DEVICE_STATE_RUNNING:
2432                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2433                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2434                         /*
2435                          * No active disks or no disks at all,
2436                          * so destroy device.
2437                          */
2438                         if (sc->sc_provider != NULL)
2439                                 g_mirror_destroy_provider(sc);
2440                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2441                         break;
2442                 } else if (g_mirror_ndisks(sc,
2443                     G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2444                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2445                         /*
2446                          * We have active disks, launch provider if it doesn't
2447                          * exist.
2448                          */
2449                         if (sc->sc_provider == NULL)
2450                                 g_mirror_launch_provider(sc);
2451                         if (sc->sc_rootmount != NULL) {
2452                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2453                                     __LINE__, sc->sc_rootmount);
2454                                 root_mount_rel(sc->sc_rootmount);
2455                                 sc->sc_rootmount = NULL;
2456                         }
2457                 }
2458                 /*
2459                  * Genid should be bumped immediately, so do it here.
2460                  */
2461                 if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2462                         sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2463                         g_mirror_bump_genid(sc);
2464                 }
2465                 break;
2466         default:
2467                 KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2468                     sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2469                 break;
2470         }
2471 }
2472
2473 /*
2474  * Update disk state and device state if needed.
2475  */
2476 #define DISK_STATE_CHANGED()    G_MIRROR_DEBUG(1,                       \
2477         "Disk %s state changed from %s to %s (device %s).",             \
2478         g_mirror_get_diskname(disk),                                    \
2479         g_mirror_disk_state2str(disk->d_state),                         \
2480         g_mirror_disk_state2str(state), sc->sc_name)
2481 static int
2482 g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2483 {
2484         struct g_mirror_softc *sc;
2485
2486         sc = disk->d_softc;
2487         sx_assert(&sc->sc_lock, SX_XLOCKED);
2488
2489 again:
2490         G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2491             g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2492             g_mirror_disk_state2str(state));
2493         switch (state) {
2494         case G_MIRROR_DISK_STATE_NEW:
2495                 /*
2496                  * Possible scenarios:
2497                  * 1. New disk arrive.
2498                  */
2499                 /* Previous state should be NONE. */
2500                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2501                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2502                     g_mirror_disk_state2str(disk->d_state)));
2503                 DISK_STATE_CHANGED();
2504
2505                 disk->d_state = state;
2506                 if (LIST_EMPTY(&sc->sc_disks))
2507                         LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2508                 else {
2509                         struct g_mirror_disk *dp;
2510
2511                         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2512                                 if (disk->d_priority >= dp->d_priority) {
2513                                         LIST_INSERT_BEFORE(dp, disk, d_next);
2514                                         dp = NULL;
2515                                         break;
2516                                 }
2517                                 if (LIST_NEXT(dp, d_next) == NULL)
2518                                         break;
2519                         }
2520                         if (dp != NULL)
2521                                 LIST_INSERT_AFTER(dp, disk, d_next);
2522                 }
2523                 G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2524                     sc->sc_name, g_mirror_get_diskname(disk));
2525                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2526                         break;
2527                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2528                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2529                     g_mirror_device_state2str(sc->sc_state),
2530                     g_mirror_get_diskname(disk),
2531                     g_mirror_disk_state2str(disk->d_state)));
2532                 state = g_mirror_determine_state(disk);
2533                 if (state != G_MIRROR_DISK_STATE_NONE)
2534                         goto again;
2535                 break;
2536         case G_MIRROR_DISK_STATE_ACTIVE:
2537                 /*
2538                  * Possible scenarios:
2539                  * 1. New disk does not need synchronization.
2540                  * 2. Synchronization process finished successfully.
2541                  */
2542                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2543                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2544                     g_mirror_device_state2str(sc->sc_state),
2545                     g_mirror_get_diskname(disk),
2546                     g_mirror_disk_state2str(disk->d_state)));
2547                 /* Previous state should be NEW or SYNCHRONIZING. */
2548                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2549                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2550                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2551                     g_mirror_disk_state2str(disk->d_state)));
2552                 DISK_STATE_CHANGED();
2553
2554                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2555                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2556                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2557                         g_mirror_sync_stop(disk, 0);
2558                 }
2559                 disk->d_state = state;
2560                 disk->d_sync.ds_offset = 0;
2561                 disk->d_sync.ds_offset_done = 0;
2562                 g_mirror_update_idle(sc, disk);
2563                 g_mirror_update_metadata(disk);
2564                 G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2565                     sc->sc_name, g_mirror_get_diskname(disk));
2566                 break;
2567         case G_MIRROR_DISK_STATE_STALE:
2568                 /*
2569                  * Possible scenarios:
2570                  * 1. Stale disk was connected.
2571                  */
2572                 /* Previous state should be NEW. */
2573                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2574                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2575                     g_mirror_disk_state2str(disk->d_state)));
2576                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2577                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2578                     g_mirror_device_state2str(sc->sc_state),
2579                     g_mirror_get_diskname(disk),
2580                     g_mirror_disk_state2str(disk->d_state)));
2581                 /*
2582                  * STALE state is only possible if device is marked
2583                  * NOAUTOSYNC.
2584                  */
2585                 KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2586                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2587                     g_mirror_device_state2str(sc->sc_state),
2588                     g_mirror_get_diskname(disk),
2589                     g_mirror_disk_state2str(disk->d_state)));
2590                 DISK_STATE_CHANGED();
2591
2592                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2593                 disk->d_state = state;
2594                 g_mirror_update_metadata(disk);
2595                 G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2596                     sc->sc_name, g_mirror_get_diskname(disk));
2597                 break;
2598         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2599                 /*
2600                  * Possible scenarios:
2601                  * 1. Disk which needs synchronization was connected.
2602                  */
2603                 /* Previous state should be NEW. */
2604                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2605                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2606                     g_mirror_disk_state2str(disk->d_state)));
2607                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2608                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2609                     g_mirror_device_state2str(sc->sc_state),
2610                     g_mirror_get_diskname(disk),
2611                     g_mirror_disk_state2str(disk->d_state)));
2612                 DISK_STATE_CHANGED();
2613
2614                 if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2615                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2616                 disk->d_state = state;
2617                 if (sc->sc_provider != NULL) {
2618                         g_mirror_sync_start(disk);
2619                         g_mirror_update_metadata(disk);
2620                 }
2621                 break;
2622         case G_MIRROR_DISK_STATE_DISCONNECTED:
2623                 /*
2624                  * Possible scenarios:
2625                  * 1. Device wasn't running yet, but disk disappear.
2626                  * 2. Disk was active and disapppear.
2627                  * 3. Disk disappear during synchronization process.
2628                  */
2629                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2630                         /*
2631                          * Previous state should be ACTIVE, STALE or
2632                          * SYNCHRONIZING.
2633                          */
2634                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2635                             disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2636                             disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2637                             ("Wrong disk state (%s, %s).",
2638                             g_mirror_get_diskname(disk),
2639                             g_mirror_disk_state2str(disk->d_state)));
2640                 } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2641                         /* Previous state should be NEW. */
2642                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2643                             ("Wrong disk state (%s, %s).",
2644                             g_mirror_get_diskname(disk),
2645                             g_mirror_disk_state2str(disk->d_state)));
2646                         /*
2647                          * Reset bumping syncid if disk disappeared in STARTING
2648                          * state.
2649                          */
2650                         if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2651                                 sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2652 #ifdef  INVARIANTS
2653                 } else {
2654                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2655                             sc->sc_name,
2656                             g_mirror_device_state2str(sc->sc_state),
2657                             g_mirror_get_diskname(disk),
2658                             g_mirror_disk_state2str(disk->d_state)));
2659 #endif
2660                 }
2661                 DISK_STATE_CHANGED();
2662                 G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2663                     sc->sc_name, g_mirror_get_diskname(disk));
2664
2665                 g_mirror_destroy_disk(disk);
2666                 break;
2667         case G_MIRROR_DISK_STATE_DESTROY:
2668             {
2669                 int error;
2670
2671                 error = g_mirror_clear_metadata(disk);
2672                 if (error != 0)
2673                         return (error);
2674                 DISK_STATE_CHANGED();
2675                 G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2676                     sc->sc_name, g_mirror_get_diskname(disk));
2677
2678                 g_mirror_destroy_disk(disk);
2679                 sc->sc_ndisks--;
2680                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2681                         g_mirror_update_metadata(disk);
2682                 }
2683                 break;
2684             }
2685         default:
2686                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2687                 break;
2688         }
2689         return (0);
2690 }
2691 #undef  DISK_STATE_CHANGED
2692
2693 int
2694 g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2695 {
2696         struct g_provider *pp;
2697         u_char *buf;
2698         int error;
2699
2700         g_topology_assert();
2701
2702         error = g_access(cp, 1, 0, 0);
2703         if (error != 0)
2704                 return (error);
2705         pp = cp->provider;
2706         g_topology_unlock();
2707         /* Metadata are stored on last sector. */
2708         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2709             &error);
2710         g_topology_lock();
2711         g_access(cp, -1, 0, 0);
2712         if (buf == NULL) {
2713                 G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2714                     cp->provider->name, error);
2715                 return (error);
2716         }
2717
2718         /* Decode metadata. */
2719         error = mirror_metadata_decode(buf, md);
2720         g_free(buf);
2721         if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2722                 return (EINVAL);
2723         if (md->md_version > G_MIRROR_VERSION) {
2724                 G_MIRROR_DEBUG(0,
2725                     "Kernel module is too old to handle metadata from %s.",
2726                     cp->provider->name);
2727                 return (EINVAL);
2728         }
2729         if (error != 0) {
2730                 G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2731                     cp->provider->name);
2732                 return (error);
2733         }
2734
2735         return (0);
2736 }
2737
2738 static int
2739 g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2740     struct g_mirror_metadata *md)
2741 {
2742
2743         if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2744                 G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2745                     pp->name, md->md_did);
2746                 return (EEXIST);
2747         }
2748         if (md->md_all != sc->sc_ndisks) {
2749                 G_MIRROR_DEBUG(1,
2750                     "Invalid '%s' field on disk %s (device %s), skipping.",
2751                     "md_all", pp->name, sc->sc_name);
2752                 return (EINVAL);
2753         }
2754         if (md->md_slice != sc->sc_slice) {
2755                 G_MIRROR_DEBUG(1,
2756                     "Invalid '%s' field on disk %s (device %s), skipping.",
2757                     "md_slice", pp->name, sc->sc_name);
2758                 return (EINVAL);
2759         }
2760         if (md->md_balance != sc->sc_balance) {
2761                 G_MIRROR_DEBUG(1,
2762                     "Invalid '%s' field on disk %s (device %s), skipping.",
2763                     "md_balance", pp->name, sc->sc_name);
2764                 return (EINVAL);
2765         }
2766 #if 0
2767         if (md->md_mediasize != sc->sc_mediasize) {
2768                 G_MIRROR_DEBUG(1,
2769                     "Invalid '%s' field on disk %s (device %s), skipping.",
2770                     "md_mediasize", pp->name, sc->sc_name);
2771                 return (EINVAL);
2772         }
2773 #endif
2774         if (sc->sc_mediasize > pp->mediasize) {
2775                 G_MIRROR_DEBUG(1,
2776                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2777                     sc->sc_name);
2778                 return (EINVAL);
2779         }
2780         if (md->md_sectorsize != sc->sc_sectorsize) {
2781                 G_MIRROR_DEBUG(1,
2782                     "Invalid '%s' field on disk %s (device %s), skipping.",
2783                     "md_sectorsize", pp->name, sc->sc_name);
2784                 return (EINVAL);
2785         }
2786         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2787                 G_MIRROR_DEBUG(1,
2788                     "Invalid sector size of disk %s (device %s), skipping.",
2789                     pp->name, sc->sc_name);
2790                 return (EINVAL);
2791         }
2792         if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2793                 G_MIRROR_DEBUG(1,
2794                     "Invalid device flags on disk %s (device %s), skipping.",
2795                     pp->name, sc->sc_name);
2796                 return (EINVAL);
2797         }
2798         if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2799                 G_MIRROR_DEBUG(1,
2800                     "Invalid disk flags on disk %s (device %s), skipping.",
2801                     pp->name, sc->sc_name);
2802                 return (EINVAL);
2803         }
2804         return (0);
2805 }
2806
2807 int
2808 g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2809     struct g_mirror_metadata *md)
2810 {
2811         struct g_mirror_disk *disk;
2812         int error;
2813
2814         g_topology_assert_not();
2815         G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2816
2817         error = g_mirror_check_metadata(sc, pp, md);
2818         if (error != 0)
2819                 return (error);
2820         if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
2821             md->md_genid < sc->sc_genid) {
2822                 G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2823                     pp->name, sc->sc_name);
2824                 return (EINVAL);
2825         }
2826         disk = g_mirror_init_disk(sc, pp, md, &error);
2827         if (disk == NULL)
2828                 return (error);
2829         error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2830             G_MIRROR_EVENT_WAIT);
2831         if (error != 0)
2832                 return (error);
2833         if (md->md_version < G_MIRROR_VERSION) {
2834                 G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2835                     pp->name, md->md_version, G_MIRROR_VERSION);
2836                 g_mirror_update_metadata(disk);
2837         }
2838         return (0);
2839 }
2840
2841 static void
2842 g_mirror_destroy_delayed(void *arg, int flag)
2843 {
2844         struct g_mirror_softc *sc;
2845         int error;
2846
2847         if (flag == EV_CANCEL) {
2848                 G_MIRROR_DEBUG(1, "Destroying canceled.");
2849                 return;
2850         }
2851         sc = arg;
2852         g_topology_unlock();
2853         sx_xlock(&sc->sc_lock);
2854         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
2855             ("DESTROY flag set on %s.", sc->sc_name));
2856         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
2857             ("DESTROYING flag not set on %s.", sc->sc_name));
2858         G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
2859         error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
2860         if (error != 0) {
2861                 G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
2862                     sc->sc_name, error);
2863                 sx_xunlock(&sc->sc_lock);
2864         }
2865         g_topology_lock();
2866 }
2867
2868 static int
2869 g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
2870 {
2871         struct g_mirror_softc *sc;
2872         int dcr, dcw, dce, error = 0;
2873
2874         g_topology_assert();
2875         G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
2876             acw, ace);
2877
2878         sc = pp->geom->softc;
2879         if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
2880                 return (0);
2881         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
2882
2883         dcr = pp->acr + acr;
2884         dcw = pp->acw + acw;
2885         dce = pp->ace + ace;
2886
2887         g_topology_unlock();
2888         sx_xlock(&sc->sc_lock);
2889         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
2890             LIST_EMPTY(&sc->sc_disks)) {
2891                 if (acr > 0 || acw > 0 || ace > 0)
2892                         error = ENXIO;
2893                 goto end;
2894         }
2895         if (dcw == 0)
2896                 g_mirror_idle(sc, dcw);
2897         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
2898                 if (acr > 0 || acw > 0 || ace > 0) {
2899                         error = ENXIO;
2900                         goto end;
2901                 }
2902                 if (dcr == 0 && dcw == 0 && dce == 0) {
2903                         g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
2904                             sc, NULL);
2905                 }
2906         }
2907 end:
2908         sx_xunlock(&sc->sc_lock);
2909         g_topology_lock();
2910         return (error);
2911 }
2912
2913 static struct g_geom *
2914 g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
2915 {
2916         struct g_mirror_softc *sc;
2917         struct g_geom *gp;
2918         int error, timeout;
2919
2920         g_topology_assert();
2921         G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
2922             md->md_mid);
2923
2924         /* One disk is minimum. */
2925         if (md->md_all < 1)
2926                 return (NULL);
2927         /*
2928          * Action geom.
2929          */
2930         gp = g_new_geomf(mp, "%s", md->md_name);
2931         sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
2932         gp->start = g_mirror_start;
2933         gp->orphan = g_mirror_orphan;
2934         gp->access = g_mirror_access;
2935         gp->dumpconf = g_mirror_dumpconf;
2936
2937         sc->sc_id = md->md_mid;
2938         sc->sc_slice = md->md_slice;
2939         sc->sc_balance = md->md_balance;
2940         sc->sc_mediasize = md->md_mediasize;
2941         sc->sc_sectorsize = md->md_sectorsize;
2942         sc->sc_ndisks = md->md_all;
2943         sc->sc_flags = md->md_mflags;
2944         sc->sc_bump_id = 0;
2945         sc->sc_idle = 1;
2946         sc->sc_last_write = time_uptime;
2947         sc->sc_writes = 0;
2948         sx_init(&sc->sc_lock, "gmirror:lock");
2949         bioq_init(&sc->sc_queue);
2950         mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
2951         bioq_init(&sc->sc_regular_delayed);
2952         bioq_init(&sc->sc_inflight);
2953         bioq_init(&sc->sc_sync_delayed);
2954         LIST_INIT(&sc->sc_disks);
2955         TAILQ_INIT(&sc->sc_events);
2956         mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
2957         callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
2958         mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
2959         sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
2960         gp->softc = sc;
2961         sc->sc_geom = gp;
2962         sc->sc_provider = NULL;
2963         /*
2964          * Synchronization geom.
2965          */
2966         gp = g_new_geomf(mp, "%s.sync", md->md_name);
2967         gp->softc = sc;
2968         gp->orphan = g_mirror_orphan;
2969         sc->sc_sync.ds_geom = gp;
2970         sc->sc_sync.ds_ndisks = 0;
2971         error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
2972             "g_mirror %s", md->md_name);
2973         if (error != 0) {
2974                 G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
2975                     sc->sc_name);
2976                 g_destroy_geom(sc->sc_sync.ds_geom);
2977                 mtx_destroy(&sc->sc_done_mtx);
2978                 mtx_destroy(&sc->sc_events_mtx);
2979                 mtx_destroy(&sc->sc_queue_mtx);
2980                 sx_destroy(&sc->sc_lock);
2981                 g_destroy_geom(sc->sc_geom);
2982                 free(sc, M_MIRROR);
2983                 return (NULL);
2984         }
2985
2986         G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
2987             sc->sc_name, sc->sc_ndisks, sc->sc_id);
2988
2989         sc->sc_rootmount = root_mount_hold("GMIRROR");
2990         G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
2991         /*
2992          * Run timeout.
2993          */
2994         timeout = g_mirror_timeout * hz;
2995         callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
2996         return (sc->sc_geom);
2997 }
2998
2999 int
3000 g_mirror_destroy(struct g_mirror_softc *sc, int how)
3001 {
3002         struct g_mirror_disk *disk;
3003         struct g_provider *pp;
3004
3005         g_topology_assert_not();
3006         if (sc == NULL)
3007                 return (ENXIO);
3008         sx_assert(&sc->sc_lock, SX_XLOCKED);
3009
3010         pp = sc->sc_provider;
3011         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
3012                 switch (how) {
3013                 case G_MIRROR_DESTROY_SOFT:
3014                         G_MIRROR_DEBUG(1,
3015                             "Device %s is still open (r%dw%de%d).", pp->name,
3016                             pp->acr, pp->acw, pp->ace);
3017                         return (EBUSY);
3018                 case G_MIRROR_DESTROY_DELAYED:
3019                         G_MIRROR_DEBUG(1,
3020                             "Device %s will be destroyed on last close.",
3021                             pp->name);
3022                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
3023                                 if (disk->d_state ==
3024                                     G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3025                                         g_mirror_sync_stop(disk, 1);
3026                                 }
3027                         }
3028                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
3029                         return (EBUSY);
3030                 case G_MIRROR_DESTROY_HARD:
3031                         G_MIRROR_DEBUG(1, "Device %s is still open, so it "
3032                             "can't be definitely removed.", pp->name);
3033                 }
3034         }
3035
3036         g_topology_lock();
3037         if (sc->sc_geom->softc == NULL) {
3038                 g_topology_unlock();
3039                 return (0);
3040         }
3041         sc->sc_geom->softc = NULL;
3042         sc->sc_sync.ds_geom->softc = NULL;
3043         g_topology_unlock();
3044
3045         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3046         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
3047         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3048         sx_xunlock(&sc->sc_lock);
3049         mtx_lock(&sc->sc_queue_mtx);
3050         wakeup(sc);
3051         mtx_unlock(&sc->sc_queue_mtx);
3052         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3053         while (sc->sc_worker != NULL)
3054                 tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3055         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3056         sx_xlock(&sc->sc_lock);
3057         g_mirror_destroy_device(sc);
3058         free(sc, M_MIRROR);
3059         return (0);
3060 }
3061
3062 static void
3063 g_mirror_taste_orphan(struct g_consumer *cp)
3064 {
3065
3066         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3067             cp->provider->name));
3068 }
3069
3070 static struct g_geom *
3071 g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3072 {
3073         struct g_mirror_metadata md;
3074         struct g_mirror_softc *sc;
3075         struct g_consumer *cp;
3076         struct g_geom *gp;
3077         int error;
3078
3079         g_topology_assert();
3080         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3081         G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3082
3083         gp = g_new_geomf(mp, "mirror:taste");
3084         /*
3085          * This orphan function should be never called.
3086          */
3087         gp->orphan = g_mirror_taste_orphan;
3088         cp = g_new_consumer(gp);
3089         g_attach(cp, pp);
3090         error = g_mirror_read_metadata(cp, &md);
3091         g_detach(cp);
3092         g_destroy_consumer(cp);
3093         g_destroy_geom(gp);
3094         if (error != 0)
3095                 return (NULL);
3096         gp = NULL;
3097
3098         if (md.md_provider[0] != '\0' &&
3099             !g_compare_names(md.md_provider, pp->name))
3100                 return (NULL);
3101         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3102                 return (NULL);
3103         if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3104                 G_MIRROR_DEBUG(0,
3105                     "Device %s: provider %s marked as inactive, skipping.",
3106                     md.md_name, pp->name);
3107                 return (NULL);
3108         }
3109         if (g_mirror_debug >= 2)
3110                 mirror_metadata_dump(&md);
3111
3112         /*
3113          * Let's check if device already exists.
3114          */
3115         sc = NULL;
3116         LIST_FOREACH(gp, &mp->geom, geom) {
3117                 sc = gp->softc;
3118                 if (sc == NULL)
3119                         continue;
3120                 if (sc->sc_sync.ds_geom == gp)
3121                         continue;
3122                 if (strcmp(md.md_name, sc->sc_name) != 0)
3123                         continue;
3124                 if (md.md_mid != sc->sc_id) {
3125                         G_MIRROR_DEBUG(0, "Device %s already configured.",
3126                             sc->sc_name);
3127                         return (NULL);
3128                 }
3129                 break;
3130         }
3131         if (gp == NULL) {
3132                 gp = g_mirror_create(mp, &md);
3133                 if (gp == NULL) {
3134                         G_MIRROR_DEBUG(0, "Cannot create device %s.",
3135                             md.md_name);
3136                         return (NULL);
3137                 }
3138                 sc = gp->softc;
3139         }
3140         G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3141         g_topology_unlock();
3142         sx_xlock(&sc->sc_lock);
3143         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3144         error = g_mirror_add_disk(sc, pp, &md);
3145         if (error != 0) {
3146                 G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3147                     pp->name, gp->name, error);
3148                 if (LIST_EMPTY(&sc->sc_disks)) {
3149                         g_cancel_event(sc);
3150                         g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3151                         g_topology_lock();
3152                         return (NULL);
3153                 }
3154                 gp = NULL;
3155         }
3156         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3157         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3158                 g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3159                 g_topology_lock();
3160                 return (NULL);
3161         }
3162         sx_xunlock(&sc->sc_lock);
3163         g_topology_lock();
3164         return (gp);
3165 }
3166
3167 static void
3168 g_mirror_resize(struct g_consumer *cp)
3169 {
3170         struct g_mirror_disk *disk;
3171
3172         g_topology_assert();
3173         g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3174
3175         disk = cp->private;
3176         if (disk == NULL)
3177                 return;
3178         g_topology_unlock();
3179         g_mirror_update_metadata(disk);
3180         g_topology_lock();
3181 }
3182
3183 static int
3184 g_mirror_destroy_geom(struct gctl_req *req __unused,
3185     struct g_class *mp __unused, struct g_geom *gp)
3186 {
3187         struct g_mirror_softc *sc;
3188         int error;
3189
3190         g_topology_unlock();
3191         sc = gp->softc;
3192         sx_xlock(&sc->sc_lock);
3193         g_cancel_event(sc);
3194         error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3195         if (error != 0)
3196                 sx_xunlock(&sc->sc_lock);
3197         g_topology_lock();
3198         return (error);
3199 }
3200
3201 static void
3202 g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3203     struct g_consumer *cp, struct g_provider *pp)
3204 {
3205         struct g_mirror_softc *sc;
3206
3207         g_topology_assert();
3208
3209         sc = gp->softc;
3210         if (sc == NULL)
3211                 return;
3212         /* Skip synchronization geom. */
3213         if (gp == sc->sc_sync.ds_geom)
3214                 return;
3215         if (pp != NULL) {
3216                 /* Nothing here. */
3217         } else if (cp != NULL) {
3218                 struct g_mirror_disk *disk;
3219
3220                 disk = cp->private;
3221                 if (disk == NULL)
3222                         return;
3223                 g_topology_unlock();
3224                 sx_xlock(&sc->sc_lock);
3225                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3226                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3227                         sbuf_printf(sb, "%s<Synchronized>", indent);
3228                         if (disk->d_sync.ds_offset == 0)
3229                                 sbuf_printf(sb, "0%%");
3230                         else {
3231                                 sbuf_printf(sb, "%u%%",
3232                                     (u_int)((disk->d_sync.ds_offset * 100) /
3233                                     sc->sc_provider->mediasize));
3234                         }
3235                         sbuf_printf(sb, "</Synchronized>\n");
3236                         if (disk->d_sync.ds_offset > 0) {
3237                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3238                                     "</BytesSynced>\n", indent,
3239                                     (intmax_t)disk->d_sync.ds_offset);
3240                         }
3241                 }
3242                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3243                     disk->d_sync.ds_syncid);
3244                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3245                     disk->d_genid);
3246                 sbuf_printf(sb, "%s<Flags>", indent);
3247                 if (disk->d_flags == 0)
3248                         sbuf_printf(sb, "NONE");
3249                 else {
3250                         int first = 1;
3251
3252 #define ADD_FLAG(flag, name)    do {                                    \
3253         if ((disk->d_flags & (flag)) != 0) {                            \
3254                 if (!first)                                             \
3255                         sbuf_printf(sb, ", ");                          \
3256                 else                                                    \
3257                         first = 0;                                      \
3258                 sbuf_printf(sb, name);                                  \
3259         }                                                               \
3260 } while (0)
3261                         ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3262                         ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3263                         ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3264                         ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3265                             "SYNCHRONIZING");
3266                         ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3267                         ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3268 #undef  ADD_FLAG
3269                 }
3270                 sbuf_printf(sb, "</Flags>\n");
3271                 sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3272                     disk->d_priority);
3273                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3274                     g_mirror_disk_state2str(disk->d_state));
3275                 sx_xunlock(&sc->sc_lock);
3276                 g_topology_lock();
3277         } else {
3278                 g_topology_unlock();
3279                 sx_xlock(&sc->sc_lock);
3280                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3281                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3282                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3283                 sbuf_printf(sb, "%s<Flags>", indent);
3284                 if (sc->sc_flags == 0)
3285                         sbuf_printf(sb, "NONE");
3286                 else {
3287                         int first = 1;
3288
3289 #define ADD_FLAG(flag, name)    do {                                    \
3290         if ((sc->sc_flags & (flag)) != 0) {                             \
3291                 if (!first)                                             \
3292                         sbuf_printf(sb, ", ");                          \
3293                 else                                                    \
3294                         first = 0;                                      \
3295                 sbuf_printf(sb, name);                                  \
3296         }                                                               \
3297 } while (0)
3298                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3299                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3300 #undef  ADD_FLAG
3301                 }
3302                 sbuf_printf(sb, "</Flags>\n");
3303                 sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3304                     (u_int)sc->sc_slice);
3305                 sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3306                     balance_name(sc->sc_balance));
3307                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3308                     sc->sc_ndisks);
3309                 sbuf_printf(sb, "%s<State>", indent);
3310                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3311                         sbuf_printf(sb, "%s", "STARTING");
3312                 else if (sc->sc_ndisks ==
3313                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3314                         sbuf_printf(sb, "%s", "COMPLETE");
3315                 else
3316                         sbuf_printf(sb, "%s", "DEGRADED");
3317                 sbuf_printf(sb, "</State>\n");
3318                 sx_xunlock(&sc->sc_lock);
3319                 g_topology_lock();
3320         }
3321 }
3322
3323 static void
3324 g_mirror_shutdown_post_sync(void *arg, int howto)
3325 {
3326         struct g_class *mp;
3327         struct g_geom *gp, *gp2;
3328         struct g_mirror_softc *sc;
3329         int error;
3330
3331         mp = arg;
3332         DROP_GIANT();
3333         g_topology_lock();
3334         g_mirror_shutdown = 1;
3335         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3336                 if ((sc = gp->softc) == NULL)
3337                         continue;
3338                 /* Skip synchronization geom. */
3339                 if (gp == sc->sc_sync.ds_geom)
3340                         continue;
3341                 g_topology_unlock();
3342                 sx_xlock(&sc->sc_lock);
3343                 g_mirror_idle(sc, -1);
3344                 g_cancel_event(sc);
3345                 error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3346                 if (error != 0)
3347                         sx_xunlock(&sc->sc_lock);
3348                 g_topology_lock();
3349         }
3350         g_topology_unlock();
3351         PICKUP_GIANT();
3352 }
3353
3354 static void
3355 g_mirror_init(struct g_class *mp)
3356 {
3357
3358         g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3359             g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3360         if (g_mirror_post_sync == NULL)
3361                 G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3362 }
3363
3364 static void
3365 g_mirror_fini(struct g_class *mp)
3366 {
3367
3368         if (g_mirror_post_sync != NULL)
3369                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3370 }
3371
3372 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);