sys/geom/mirror/g_mirror.c

   1 /*-
   2  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/module.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/bio.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/malloc.h>
  41 #include <sys/eventhandler.h>
  42 #include <vm/uma.h>
  43 #include <geom/geom.h>
  44 #include <sys/proc.h>
  45 #include <sys/kthread.h>
  46 #include <sys/sched.h>
  47 #include <geom/mirror/g_mirror.h>
  48
  49 FEATURE(geom_mirror, "GEOM mirroring support");
  50
  51 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
  52
  53 SYSCTL_DECL(_kern_geom);
  54 static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
  55     "GEOM_MIRROR stuff");
  56 u_int g_mirror_debug = 0;
  57 TUNABLE_INT("kern.geom.mirror.debug", &g_mirror_debug);
  58 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RW, &g_mirror_debug, 0,
  59     "Debug level");
  60 static u_int g_mirror_timeout = 4;
  61 TUNABLE_INT("kern.geom.mirror.timeout", &g_mirror_timeout);
  62 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RW, &g_mirror_timeout,
  63     0, "Time to wait on all mirror components");
  64 static u_int g_mirror_idletime = 5;
  65 TUNABLE_INT("kern.geom.mirror.idletime", &g_mirror_idletime);
  66 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RW,
  67     &g_mirror_idletime, 0, "Mark components as clean when idling");
  68 static u_int g_mirror_disconnect_on_failure = 1;
  69 TUNABLE_INT("kern.geom.mirror.disconnect_on_failure",
  70     &g_mirror_disconnect_on_failure);
  71 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RW,
  72     &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  73 static u_int g_mirror_syncreqs = 2;
  74 TUNABLE_INT("kern.geom.mirror.sync_requests", &g_mirror_syncreqs);
  75 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  76     &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
  77
  78 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  79         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));       \
  80         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  81         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));       \
  82 } while (0)
  83
  84 static eventhandler_tag g_mirror_post_sync = NULL;
  85 static int g_mirror_shutdown = 0;
  86
  87 static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
  88     struct g_geom *gp);
  89 static g_taste_t g_mirror_taste;
  90 static g_resize_t g_mirror_resize;
  91 static void g_mirror_init(struct g_class *mp);
  92 static void g_mirror_fini(struct g_class *mp);
  93
  94 struct g_class g_mirror_class = {
  95         .name = G_MIRROR_CLASS_NAME,
  96         .version = G_VERSION,
  97         .ctlreq = g_mirror_config,
  98         .taste = g_mirror_taste,
  99         .destroy_geom = g_mirror_destroy_geom,
 100         .init = g_mirror_init,
 101         .fini = g_mirror_fini,
 102         .resize = g_mirror_resize
 103 };
 104
 105
 106 static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
 107 static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
 108 static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
 109 static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
 110     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 111 static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
 112 static void g_mirror_register_request(struct bio *bp);
 113 static void g_mirror_sync_release(struct g_mirror_softc *sc);
 114
 115
 116 static const char *
 117 g_mirror_disk_state2str(int state)
 118 {
 119
 120         switch (state) {
 121         case G_MIRROR_DISK_STATE_NONE:
 122                 return ("NONE");
 123         case G_MIRROR_DISK_STATE_NEW:
 124                 return ("NEW");
 125         case G_MIRROR_DISK_STATE_ACTIVE:
 126                 return ("ACTIVE");
 127         case G_MIRROR_DISK_STATE_STALE:
 128                 return ("STALE");
 129         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 130                 return ("SYNCHRONIZING");
 131         case G_MIRROR_DISK_STATE_DISCONNECTED:
 132                 return ("DISCONNECTED");
 133         case G_MIRROR_DISK_STATE_DESTROY:
 134                 return ("DESTROY");
 135         default:
 136                 return ("INVALID");
 137         }
 138 }
 139
 140 static const char *
 141 g_mirror_device_state2str(int state)
 142 {
 143
 144         switch (state) {
 145         case G_MIRROR_DEVICE_STATE_STARTING:
 146                 return ("STARTING");
 147         case G_MIRROR_DEVICE_STATE_RUNNING:
 148                 return ("RUNNING");
 149         default:
 150                 return ("INVALID");
 151         }
 152 }
 153
 154 static const char *
 155 g_mirror_get_diskname(struct g_mirror_disk *disk)
 156 {
 157
 158         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 159                 return ("[unknown]");
 160         return (disk->d_name);
 161 }
 162
 163 /*
 164  * --- Events handling functions ---
 165  * Events in geom_mirror are used to maintain disks and device status
 166  * from one thread to simplify locking.
 167  */
 168 static void
 169 g_mirror_event_free(struct g_mirror_event *ep)
 170 {
 171
 172         free(ep, M_MIRROR);
 173 }
 174
 175 int
 176 g_mirror_event_send(void *arg, int state, int flags)
 177 {
 178         struct g_mirror_softc *sc;
 179         struct g_mirror_disk *disk;
 180         struct g_mirror_event *ep;
 181         int error;
 182
 183         ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
 184         G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 185         if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
 186                 disk = NULL;
 187                 sc = arg;
 188         } else {
 189                 disk = arg;
 190                 sc = disk->d_softc;
 191         }
 192         ep->e_disk = disk;
 193         ep->e_state = state;
 194         ep->e_flags = flags;
 195         ep->e_error = 0;
 196         mtx_lock(&sc->sc_events_mtx);
 197         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 198         mtx_unlock(&sc->sc_events_mtx);
 199         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 200         mtx_lock(&sc->sc_queue_mtx);
 201         wakeup(sc);
 202         mtx_unlock(&sc->sc_queue_mtx);
 203         if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 204                 return (0);
 205         sx_assert(&sc->sc_lock, SX_XLOCKED);
 206         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 207         sx_xunlock(&sc->sc_lock);
 208         while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
 209                 mtx_lock(&sc->sc_events_mtx);
 210                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
 211                     hz * 5);
 212         }
 213         error = ep->e_error;
 214         g_mirror_event_free(ep);
 215         sx_xlock(&sc->sc_lock);
 216         return (error);
 217 }
 218
 219 static struct g_mirror_event *
 220 g_mirror_event_get(struct g_mirror_softc *sc)
 221 {
 222         struct g_mirror_event *ep;
 223
 224         mtx_lock(&sc->sc_events_mtx);
 225         ep = TAILQ_FIRST(&sc->sc_events);
 226         mtx_unlock(&sc->sc_events_mtx);
 227         return (ep);
 228 }
 229
 230 static void
 231 g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
 232 {
 233
 234         mtx_lock(&sc->sc_events_mtx);
 235         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 236         mtx_unlock(&sc->sc_events_mtx);
 237 }
 238
 239 static void
 240 g_mirror_event_cancel(struct g_mirror_disk *disk)
 241 {
 242         struct g_mirror_softc *sc;
 243         struct g_mirror_event *ep, *tmpep;
 244
 245         sc = disk->d_softc;
 246         sx_assert(&sc->sc_lock, SX_XLOCKED);
 247
 248         mtx_lock(&sc->sc_events_mtx);
 249         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 250                 if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
 251                         continue;
 252                 if (ep->e_disk != disk)
 253                         continue;
 254                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 255                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 256                         g_mirror_event_free(ep);
 257                 else {
 258                         ep->e_error = ECANCELED;
 259                         wakeup(ep);
 260                 }
 261         }
 262         mtx_unlock(&sc->sc_events_mtx);
 263 }
 264
 265 /*
 266  * Return the number of disks in given state.
 267  * If state is equal to -1, count all connected disks.
 268  */
 269 u_int
 270 g_mirror_ndisks(struct g_mirror_softc *sc, int state)
 271 {
 272         struct g_mirror_disk *disk;
 273         u_int n = 0;
 274
 275         sx_assert(&sc->sc_lock, SX_LOCKED);
 276
 277         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 278                 if (state == -1 || disk->d_state == state)
 279                         n++;
 280         }
 281         return (n);
 282 }
 283
 284 /*
 285  * Find a disk in mirror by its disk ID.
 286  */
 287 static struct g_mirror_disk *
 288 g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
 289 {
 290         struct g_mirror_disk *disk;
 291
 292         sx_assert(&sc->sc_lock, SX_XLOCKED);
 293
 294         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 295                 if (disk->d_id == id)
 296                         return (disk);
 297         }
 298         return (NULL);
 299 }
 300
 301 static u_int
 302 g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
 303 {
 304         struct bio *bp;
 305         u_int nreqs = 0;
 306
 307         mtx_lock(&sc->sc_queue_mtx);
 308         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 309                 if (bp->bio_from == cp)
 310                         nreqs++;
 311         }
 312         mtx_unlock(&sc->sc_queue_mtx);
 313         return (nreqs);
 314 }
 315
 316 static int
 317 g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
 318 {
 319
 320         if (cp->index > 0) {
 321                 G_MIRROR_DEBUG(2,
 322                     "I/O requests for %s exist, can't destroy it now.",
 323                     cp->provider->name);
 324                 return (1);
 325         }
 326         if (g_mirror_nrequests(sc, cp) > 0) {
 327                 G_MIRROR_DEBUG(2,
 328                     "I/O requests for %s in queue, can't destroy it now.",
 329                     cp->provider->name);
 330                 return (1);
 331         }
 332         return (0);
 333 }
 334
 335 static void
 336 g_mirror_destroy_consumer(void *arg, int flags __unused)
 337 {
 338         struct g_consumer *cp;
 339
 340         g_topology_assert();
 341
 342         cp = arg;
 343         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 344         g_detach(cp);
 345         g_destroy_consumer(cp);
 346 }
 347
 348 static void
 349 g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 350 {
 351         struct g_provider *pp;
 352         int retaste_wait;
 353
 354         g_topology_assert();
 355
 356         cp->private = NULL;
 357         if (g_mirror_is_busy(sc, cp))
 358                 return;
 359         pp = cp->provider;
 360         retaste_wait = 0;
 361         if (cp->acw == 1) {
 362                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 363                         retaste_wait = 1;
 364         }
 365         G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 366             -cp->acw, -cp->ace, 0);
 367         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 368                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 369         if (retaste_wait) {
 370                 /*
 371                  * After retaste event was send (inside g_access()), we can send
 372                  * event to detach and destroy consumer.
 373                  * A class, which has consumer to the given provider connected
 374                  * will not receive retaste event for the provider.
 375                  * This is the way how I ignore retaste events when I close
 376                  * consumers opened for write: I detach and destroy consumer
 377                  * after retaste event is sent.
 378                  */
 379                 g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
 380                 return;
 381         }
 382         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
 383         g_detach(cp);
 384         g_destroy_consumer(cp);
 385 }
 386
 387 static int
 388 g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
 389 {
 390         struct g_consumer *cp;
 391         int error;
 392
 393         g_topology_assert_not();
 394         KASSERT(disk->d_consumer == NULL,
 395             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 396
 397         g_topology_lock();
 398         cp = g_new_consumer(disk->d_softc->sc_geom);
 399         cp->flags |= G_CF_DIRECT_RECEIVE;
 400         error = g_attach(cp, pp);
 401         if (error != 0) {
 402                 g_destroy_consumer(cp);
 403                 g_topology_unlock();
 404                 return (error);
 405         }
 406         error = g_access(cp, 1, 1, 1);
 407         if (error != 0) {
 408                 g_detach(cp);
 409                 g_destroy_consumer(cp);
 410                 g_topology_unlock();
 411                 G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
 412                     pp->name, error);
 413                 return (error);
 414         }
 415         g_topology_unlock();
 416         disk->d_consumer = cp;
 417         disk->d_consumer->private = disk;
 418         disk->d_consumer->index = 0;
 419
 420         G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
 421         return (0);
 422 }
 423
 424 static void
 425 g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 426 {
 427
 428         g_topology_assert();
 429
 430         if (cp == NULL)
 431                 return;
 432         if (cp->provider != NULL)
 433                 g_mirror_kill_consumer(sc, cp);
 434         else
 435                 g_destroy_consumer(cp);
 436 }
 437
 438 /*
 439  * Initialize disk. This means allocate memory, create consumer, attach it
 440  * to the provider and open access (r1w1e1) to it.
 441  */
 442 static struct g_mirror_disk *
 443 g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 444     struct g_mirror_metadata *md, int *errorp)
 445 {
 446         struct g_mirror_disk *disk;
 447         int i, error;
 448
 449         disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
 450         if (disk == NULL) {
 451                 error = ENOMEM;
 452                 goto fail;
 453         }
 454         disk->d_softc = sc;
 455         error = g_mirror_connect_disk(disk, pp);
 456         if (error != 0)
 457                 goto fail;
 458         disk->d_id = md->md_did;
 459         disk->d_state = G_MIRROR_DISK_STATE_NONE;
 460         disk->d_priority = md->md_priority;
 461         disk->d_flags = md->md_dflags;
 462         error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
 463         if (error == 0 && i != 0)
 464                 disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 465         if (md->md_provider[0] != '\0')
 466                 disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
 467         disk->d_sync.ds_consumer = NULL;
 468         disk->d_sync.ds_offset = md->md_sync_offset;
 469         disk->d_sync.ds_offset_done = md->md_sync_offset;
 470         disk->d_genid = md->md_genid;
 471         disk->d_sync.ds_syncid = md->md_syncid;
 472         if (errorp != NULL)
 473                 *errorp = 0;
 474         return (disk);
 475 fail:
 476         if (errorp != NULL)
 477                 *errorp = error;
 478         if (disk != NULL)
 479                 free(disk, M_MIRROR);
 480         return (NULL);
 481 }
 482
 483 static void
 484 g_mirror_destroy_disk(struct g_mirror_disk *disk)
 485 {
 486         struct g_mirror_softc *sc;
 487
 488         g_topology_assert_not();
 489         sc = disk->d_softc;
 490         sx_assert(&sc->sc_lock, SX_XLOCKED);
 491
 492         LIST_REMOVE(disk, d_next);
 493         g_mirror_event_cancel(disk);
 494         if (sc->sc_hint == disk)
 495                 sc->sc_hint = NULL;
 496         switch (disk->d_state) {
 497         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 498                 g_mirror_sync_stop(disk, 1);
 499                 /* FALLTHROUGH */
 500         case G_MIRROR_DISK_STATE_NEW:
 501         case G_MIRROR_DISK_STATE_STALE:
 502         case G_MIRROR_DISK_STATE_ACTIVE:
 503                 g_topology_lock();
 504                 g_mirror_disconnect_consumer(sc, disk->d_consumer);
 505                 g_topology_unlock();
 506                 free(disk, M_MIRROR);
 507                 break;
 508         default:
 509                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 510                     g_mirror_get_diskname(disk),
 511                     g_mirror_disk_state2str(disk->d_state)));
 512         }
 513 }
 514
 515 static void
 516 g_mirror_destroy_device(struct g_mirror_softc *sc)
 517 {
 518         struct g_mirror_disk *disk;
 519         struct g_mirror_event *ep;
 520         struct g_geom *gp;
 521         struct g_consumer *cp, *tmpcp;
 522
 523         g_topology_assert_not();
 524         sx_assert(&sc->sc_lock, SX_XLOCKED);
 525
 526         gp = sc->sc_geom;
 527         if (sc->sc_provider != NULL)
 528                 g_mirror_destroy_provider(sc);
 529         for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
 530             disk = LIST_FIRST(&sc->sc_disks)) {
 531                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 532                 g_mirror_update_metadata(disk);
 533                 g_mirror_destroy_disk(disk);
 534         }
 535         while ((ep = g_mirror_event_get(sc)) != NULL) {
 536                 g_mirror_event_remove(sc, ep);
 537                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 538                         g_mirror_event_free(ep);
 539                 else {
 540                         ep->e_error = ECANCELED;
 541                         ep->e_flags |= G_MIRROR_EVENT_DONE;
 542                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 543                         mtx_lock(&sc->sc_events_mtx);
 544                         wakeup(ep);
 545                         mtx_unlock(&sc->sc_events_mtx);
 546                 }
 547         }
 548         callout_drain(&sc->sc_callout);
 549
 550         g_topology_lock();
 551         LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
 552                 g_mirror_disconnect_consumer(sc, cp);
 553         }
 554         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 555         G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
 556         g_wither_geom(gp, ENXIO);
 557         g_topology_unlock();
 558         mtx_destroy(&sc->sc_queue_mtx);
 559         mtx_destroy(&sc->sc_events_mtx);
 560         mtx_destroy(&sc->sc_done_mtx);
 561         sx_xunlock(&sc->sc_lock);
 562         sx_destroy(&sc->sc_lock);
 563 }
 564
 565 static void
 566 g_mirror_orphan(struct g_consumer *cp)
 567 {
 568         struct g_mirror_disk *disk;
 569
 570         g_topology_assert();
 571
 572         disk = cp->private;
 573         if (disk == NULL)
 574                 return;
 575         disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 576         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 577             G_MIRROR_EVENT_DONTWAIT);
 578 }
 579
 580 /*
 581  * Function should return the next active disk on the list.
 582  * It is possible that it will be the same disk as given.
 583  * If there are no active disks on list, NULL is returned.
 584  */
 585 static __inline struct g_mirror_disk *
 586 g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
 587 {
 588         struct g_mirror_disk *dp;
 589
 590         for (dp = LIST_NEXT(disk, d_next); dp != disk;
 591             dp = LIST_NEXT(dp, d_next)) {
 592                 if (dp == NULL)
 593                         dp = LIST_FIRST(&sc->sc_disks);
 594                 if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
 595                         break;
 596         }
 597         if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 598                 return (NULL);
 599         return (dp);
 600 }
 601
 602 static struct g_mirror_disk *
 603 g_mirror_get_disk(struct g_mirror_softc *sc)
 604 {
 605         struct g_mirror_disk *disk;
 606
 607         if (sc->sc_hint == NULL) {
 608                 sc->sc_hint = LIST_FIRST(&sc->sc_disks);
 609                 if (sc->sc_hint == NULL)
 610                         return (NULL);
 611         }
 612         disk = sc->sc_hint;
 613         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
 614                 disk = g_mirror_find_next(sc, disk);
 615                 if (disk == NULL)
 616                         return (NULL);
 617         }
 618         sc->sc_hint = g_mirror_find_next(sc, disk);
 619         return (disk);
 620 }
 621
 622 static int
 623 g_mirror_write_metadata(struct g_mirror_disk *disk,
 624     struct g_mirror_metadata *md)
 625 {
 626         struct g_mirror_softc *sc;
 627         struct g_consumer *cp;
 628         off_t offset, length;
 629         u_char *sector;
 630         int error = 0;
 631
 632         g_topology_assert_not();
 633         sc = disk->d_softc;
 634         sx_assert(&sc->sc_lock, SX_LOCKED);
 635
 636         cp = disk->d_consumer;
 637         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 638         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 639         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 640             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 641             cp->acw, cp->ace));
 642         length = cp->provider->sectorsize;
 643         offset = cp->provider->mediasize - length;
 644         sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
 645         if (md != NULL &&
 646             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
 647                 /*
 648                  * Handle the case, when the size of parent provider reduced.
 649                  */
 650                 if (offset < md->md_mediasize)
 651                         error = ENOSPC;
 652                 else
 653                         mirror_metadata_encode(md, sector);
 654         }
 655         if (error == 0)
 656                 error = g_write_data(cp, offset, sector, length);
 657         free(sector, M_MIRROR);
 658         if (error != 0) {
 659                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 660                         disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 661                         G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
 662                             "(device=%s, error=%d).",
 663                             g_mirror_get_diskname(disk), sc->sc_name, error);
 664                 } else {
 665                         G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
 666                             "(device=%s, error=%d).",
 667                             g_mirror_get_diskname(disk), sc->sc_name, error);
 668                 }
 669                 if (g_mirror_disconnect_on_failure &&
 670                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 671                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 672                         g_mirror_event_send(disk,
 673                             G_MIRROR_DISK_STATE_DISCONNECTED,
 674                             G_MIRROR_EVENT_DONTWAIT);
 675                 }
 676         }
 677         return (error);
 678 }
 679
 680 static int
 681 g_mirror_clear_metadata(struct g_mirror_disk *disk)
 682 {
 683         int error;
 684
 685         g_topology_assert_not();
 686         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 687
 688         error = g_mirror_write_metadata(disk, NULL);
 689         if (error == 0) {
 690                 G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
 691                     g_mirror_get_diskname(disk));
 692         } else {
 693                 G_MIRROR_DEBUG(0,
 694                     "Cannot clear metadata on disk %s (error=%d).",
 695                     g_mirror_get_diskname(disk), error);
 696         }
 697         return (error);
 698 }
 699
 700 void
 701 g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
 702     struct g_mirror_metadata *md)
 703 {
 704
 705         strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
 706         md->md_version = G_MIRROR_VERSION;
 707         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 708         md->md_mid = sc->sc_id;
 709         md->md_all = sc->sc_ndisks;
 710         md->md_slice = sc->sc_slice;
 711         md->md_balance = sc->sc_balance;
 712         md->md_genid = sc->sc_genid;
 713         md->md_mediasize = sc->sc_mediasize;
 714         md->md_sectorsize = sc->sc_sectorsize;
 715         md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
 716         bzero(md->md_provider, sizeof(md->md_provider));
 717         if (disk == NULL) {
 718                 md->md_did = arc4random();
 719                 md->md_priority = 0;
 720                 md->md_syncid = 0;
 721                 md->md_dflags = 0;
 722                 md->md_sync_offset = 0;
 723                 md->md_provsize = 0;
 724         } else {
 725                 md->md_did = disk->d_id;
 726                 md->md_priority = disk->d_priority;
 727                 md->md_syncid = disk->d_sync.ds_syncid;
 728                 md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
 729                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
 730                         md->md_sync_offset = disk->d_sync.ds_offset_done;
 731                 else
 732                         md->md_sync_offset = 0;
 733                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
 734                         strlcpy(md->md_provider,
 735                             disk->d_consumer->provider->name,
 736                             sizeof(md->md_provider));
 737                 }
 738                 md->md_provsize = disk->d_consumer->provider->mediasize;
 739         }
 740 }
 741
 742 void
 743 g_mirror_update_metadata(struct g_mirror_disk *disk)
 744 {
 745         struct g_mirror_softc *sc;
 746         struct g_mirror_metadata md;
 747         int error;
 748
 749         g_topology_assert_not();
 750         sc = disk->d_softc;
 751         sx_assert(&sc->sc_lock, SX_LOCKED);
 752
 753         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
 754                 g_mirror_fill_metadata(sc, disk, &md);
 755         error = g_mirror_write_metadata(disk, &md);
 756         if (error == 0) {
 757                 G_MIRROR_DEBUG(2, "Metadata on %s updated.",
 758                     g_mirror_get_diskname(disk));
 759         } else {
 760                 G_MIRROR_DEBUG(0,
 761                     "Cannot update metadata on disk %s (error=%d).",
 762                     g_mirror_get_diskname(disk), error);
 763         }
 764 }
 765
 766 static void
 767 g_mirror_bump_syncid(struct g_mirror_softc *sc)
 768 {
 769         struct g_mirror_disk *disk;
 770
 771         g_topology_assert_not();
 772         sx_assert(&sc->sc_lock, SX_XLOCKED);
 773         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 774             ("%s called with no active disks (device=%s).", __func__,
 775             sc->sc_name));
 776
 777         sc->sc_syncid++;
 778         G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 779             sc->sc_syncid);
 780         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 781                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 782                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 783                         disk->d_sync.ds_syncid = sc->sc_syncid;
 784                         g_mirror_update_metadata(disk);
 785                 }
 786         }
 787 }
 788
 789 static void
 790 g_mirror_bump_genid(struct g_mirror_softc *sc)
 791 {
 792         struct g_mirror_disk *disk;
 793
 794         g_topology_assert_not();
 795         sx_assert(&sc->sc_lock, SX_XLOCKED);
 796         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 797             ("%s called with no active disks (device=%s).", __func__,
 798             sc->sc_name));
 799
 800         sc->sc_genid++;
 801         G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 802             sc->sc_genid);
 803         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 804                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 805                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 806                         disk->d_genid = sc->sc_genid;
 807                         g_mirror_update_metadata(disk);
 808                 }
 809         }
 810 }
 811
 812 static int
 813 g_mirror_idle(struct g_mirror_softc *sc, int acw)
 814 {
 815         struct g_mirror_disk *disk;
 816         int timeout;
 817
 818         g_topology_assert_not();
 819         sx_assert(&sc->sc_lock, SX_XLOCKED);
 820
 821         if (sc->sc_provider == NULL)
 822                 return (0);
 823         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 824                 return (0);
 825         if (sc->sc_idle)
 826                 return (0);
 827         if (sc->sc_writes > 0)
 828                 return (0);
 829         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 830                 timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
 831                 if (!g_mirror_shutdown && timeout > 0)
 832                         return (timeout);
 833         }
 834         sc->sc_idle = 1;
 835         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 836                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 837                         continue;
 838                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
 839                     g_mirror_get_diskname(disk), sc->sc_name);
 840                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 841                 g_mirror_update_metadata(disk);
 842         }
 843         return (0);
 844 }
 845
 846 static void
 847 g_mirror_unidle(struct g_mirror_softc *sc)
 848 {
 849         struct g_mirror_disk *disk;
 850
 851         g_topology_assert_not();
 852         sx_assert(&sc->sc_lock, SX_XLOCKED);
 853
 854         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 855                 return;
 856         sc->sc_idle = 0;
 857         sc->sc_last_write = time_uptime;
 858         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 859                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 860                         continue;
 861                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
 862                     g_mirror_get_diskname(disk), sc->sc_name);
 863                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
 864                 g_mirror_update_metadata(disk);
 865         }
 866 }
 867
 868 static void
 869 g_mirror_flush_done(struct bio *bp)
 870 {
 871         struct g_mirror_softc *sc;
 872         struct bio *pbp;
 873
 874         pbp = bp->bio_parent;
 875         sc = pbp->bio_to->geom->softc;
 876         mtx_lock(&sc->sc_done_mtx);
 877         if (pbp->bio_error == 0)
 878                 pbp->bio_error = bp->bio_error;
 879         pbp->bio_completed += bp->bio_completed;
 880         pbp->bio_inbed++;
 881         if (pbp->bio_children == pbp->bio_inbed) {
 882                 mtx_unlock(&sc->sc_done_mtx);
 883                 g_io_deliver(pbp, pbp->bio_error);
 884         } else
 885                 mtx_unlock(&sc->sc_done_mtx);
 886         g_destroy_bio(bp);
 887 }
 888
 889 static void
 890 g_mirror_done(struct bio *bp)
 891 {
 892         struct g_mirror_softc *sc;
 893
 894         sc = bp->bio_from->geom->softc;
 895         bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
 896         mtx_lock(&sc->sc_queue_mtx);
 897         bioq_insert_tail(&sc->sc_queue, bp);
 898         mtx_unlock(&sc->sc_queue_mtx);
 899         wakeup(sc);
 900 }
 901
 902 static void
 903 g_mirror_regular_request(struct bio *bp)
 904 {
 905         struct g_mirror_softc *sc;
 906         struct g_mirror_disk *disk;
 907         struct bio *pbp;
 908
 909         g_topology_assert_not();
 910
 911         pbp = bp->bio_parent;
 912         sc = pbp->bio_to->geom->softc;
 913         bp->bio_from->index--;
 914         if (bp->bio_cmd == BIO_WRITE)
 915                 sc->sc_writes--;
 916         disk = bp->bio_from->private;
 917         if (disk == NULL) {
 918                 g_topology_lock();
 919                 g_mirror_kill_consumer(sc, bp->bio_from);
 920                 g_topology_unlock();
 921         }
 922
 923         pbp->bio_inbed++;
 924         KASSERT(pbp->bio_inbed <= pbp->bio_children,
 925             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
 926             pbp->bio_children));
 927         if (bp->bio_error == 0 && pbp->bio_error == 0) {
 928                 G_MIRROR_LOGREQ(3, bp, "Request delivered.");
 929                 g_destroy_bio(bp);
 930                 if (pbp->bio_children == pbp->bio_inbed) {
 931                         G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
 932                         pbp->bio_completed = pbp->bio_length;
 933                         if (pbp->bio_cmd == BIO_WRITE ||
 934                             pbp->bio_cmd == BIO_DELETE) {
 935                                 bioq_remove(&sc->sc_inflight, pbp);
 936                                 /* Release delayed sync requests if possible. */
 937                                 g_mirror_sync_release(sc);
 938                         }
 939                         g_io_deliver(pbp, pbp->bio_error);
 940                 }
 941                 return;
 942         } else if (bp->bio_error != 0) {
 943                 if (pbp->bio_error == 0)
 944                         pbp->bio_error = bp->bio_error;
 945                 if (disk != NULL) {
 946                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 947                                 disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 948                                 G_MIRROR_LOGREQ(0, bp,
 949                                     "Request failed (error=%d).",
 950                                     bp->bio_error);
 951                         } else {
 952                                 G_MIRROR_LOGREQ(1, bp,
 953                                     "Request failed (error=%d).",
 954                                     bp->bio_error);
 955                         }
 956                         if (g_mirror_disconnect_on_failure &&
 957                             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
 958                         {
 959                                 sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 960                                 g_mirror_event_send(disk,
 961                                     G_MIRROR_DISK_STATE_DISCONNECTED,
 962                                     G_MIRROR_EVENT_DONTWAIT);
 963                         }
 964                 }
 965                 switch (pbp->bio_cmd) {
 966                 case BIO_DELETE:
 967                 case BIO_WRITE:
 968                         pbp->bio_inbed--;
 969                         pbp->bio_children--;
 970                         break;
 971                 }
 972         }
 973         g_destroy_bio(bp);
 974
 975         switch (pbp->bio_cmd) {
 976         case BIO_READ:
 977                 if (pbp->bio_inbed < pbp->bio_children)
 978                         break;
 979                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
 980                         g_io_deliver(pbp, pbp->bio_error);
 981                 else {
 982                         pbp->bio_error = 0;
 983                         mtx_lock(&sc->sc_queue_mtx);
 984                         bioq_insert_tail(&sc->sc_queue, pbp);
 985                         mtx_unlock(&sc->sc_queue_mtx);
 986                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 987                         wakeup(sc);
 988                 }
 989                 break;
 990         case BIO_DELETE:
 991         case BIO_WRITE:
 992                 if (pbp->bio_children == 0) {
 993                         /*
 994                          * All requests failed.
 995                          */
 996                 } else if (pbp->bio_inbed < pbp->bio_children) {
 997                         /* Do nothing. */
 998                         break;
 999                 } else if (pbp->bio_children == pbp->bio_inbed) {
1000                         /* Some requests succeeded. */
1001                         pbp->bio_error = 0;
1002                         pbp->bio_completed = pbp->bio_length;
1003                 }
1004                 bioq_remove(&sc->sc_inflight, pbp);
1005                 /* Release delayed sync requests if possible. */
1006                 g_mirror_sync_release(sc);
1007                 g_io_deliver(pbp, pbp->bio_error);
1008                 break;
1009         default:
1010                 KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1011                 break;
1012         }
1013 }
1014
1015 static void
1016 g_mirror_sync_done(struct bio *bp)
1017 {
1018         struct g_mirror_softc *sc;
1019
1020         G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1021         sc = bp->bio_from->geom->softc;
1022         bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1023         mtx_lock(&sc->sc_queue_mtx);
1024         bioq_insert_tail(&sc->sc_queue, bp);
1025         mtx_unlock(&sc->sc_queue_mtx);
1026         wakeup(sc);
1027 }
1028
1029 static void
1030 g_mirror_candelete(struct bio *bp)
1031 {
1032         struct g_mirror_softc *sc;
1033         struct g_mirror_disk *disk;
1034         int *val;
1035
1036         sc = bp->bio_to->geom->softc;
1037         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1038                 if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
1039                         break;
1040         }
1041         val = (int *)bp->bio_data;
1042         *val = (disk != NULL);
1043         g_io_deliver(bp, 0);
1044 }
1045
1046 static void
1047 g_mirror_kernel_dump(struct bio *bp)
1048 {
1049         struct g_mirror_softc *sc;
1050         struct g_mirror_disk *disk;
1051         struct bio *cbp;
1052         struct g_kerneldump *gkd;
1053
1054         /*
1055          * We configure dumping to the first component, because this component
1056          * will be used for reading with 'prefer' balance algorithm.
1057          * If the component with the higest priority is currently disconnected
1058          * we will not be able to read the dump after the reboot if it will be
1059          * connected and synchronized later. Can we do something better?
1060          */
1061         sc = bp->bio_to->geom->softc;
1062         disk = LIST_FIRST(&sc->sc_disks);
1063
1064         gkd = (struct g_kerneldump *)bp->bio_data;
1065         if (gkd->length > bp->bio_to->mediasize)
1066                 gkd->length = bp->bio_to->mediasize;
1067         cbp = g_clone_bio(bp);
1068         if (cbp == NULL) {
1069                 g_io_deliver(bp, ENOMEM);
1070                 return;
1071         }
1072         cbp->bio_done = g_std_done;
1073         g_io_request(cbp, disk->d_consumer);
1074         G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1075             g_mirror_get_diskname(disk));
1076 }
1077
1078 static void
1079 g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
1080 {
1081         struct bio_queue_head queue;
1082         struct g_mirror_disk *disk;
1083         struct g_consumer *cp;
1084         struct bio *cbp;
1085
1086         bioq_init(&queue);
1087         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1088                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1089                         continue;
1090                 cbp = g_clone_bio(bp);
1091                 if (cbp == NULL) {
1092                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1093                                 g_destroy_bio(cbp);
1094                         if (bp->bio_error == 0)
1095                                 bp->bio_error = ENOMEM;
1096                         g_io_deliver(bp, bp->bio_error);
1097                         return;
1098                 }
1099                 bioq_insert_tail(&queue, cbp);
1100                 cbp->bio_done = g_mirror_flush_done;
1101                 cbp->bio_caller1 = disk;
1102                 cbp->bio_to = disk->d_consumer->provider;
1103         }
1104         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1105                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1106                 disk = cbp->bio_caller1;
1107                 cbp->bio_caller1 = NULL;
1108                 cp = disk->d_consumer;
1109                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1110                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1111                     cp->acr, cp->acw, cp->ace));
1112                 g_io_request(cbp, disk->d_consumer);
1113         }
1114 }
1115
1116 static void
1117 g_mirror_start(struct bio *bp)
1118 {
1119         struct g_mirror_softc *sc;
1120
1121         sc = bp->bio_to->geom->softc;
1122         /*
1123          * If sc == NULL or there are no valid disks, provider's error
1124          * should be set and g_mirror_start() should not be called at all.
1125          */
1126         KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1127             ("Provider's error should be set (error=%d)(mirror=%s).",
1128             bp->bio_to->error, bp->bio_to->name));
1129         G_MIRROR_LOGREQ(3, bp, "Request received.");
1130
1131         switch (bp->bio_cmd) {
1132         case BIO_READ:
1133         case BIO_WRITE:
1134         case BIO_DELETE:
1135                 break;
1136         case BIO_FLUSH:
1137                 g_mirror_flush(sc, bp);
1138                 return;
1139         case BIO_GETATTR:
1140                 if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
1141                         g_mirror_candelete(bp);
1142                         return;
1143                 } else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1144                         g_mirror_kernel_dump(bp);
1145                         return;
1146                 }
1147                 /* FALLTHROUGH */
1148         default:
1149                 g_io_deliver(bp, EOPNOTSUPP);
1150                 return;
1151         }
1152         mtx_lock(&sc->sc_queue_mtx);
1153         bioq_insert_tail(&sc->sc_queue, bp);
1154         mtx_unlock(&sc->sc_queue_mtx);
1155         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1156         wakeup(sc);
1157 }
1158
1159 /*
1160  * Return TRUE if the given request is colliding with a in-progress
1161  * synchronization request.
1162  */
1163 static int
1164 g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1165 {
1166         struct g_mirror_disk *disk;
1167         struct bio *sbp;
1168         off_t rstart, rend, sstart, send;
1169         int i;
1170
1171         if (sc->sc_sync.ds_ndisks == 0)
1172                 return (0);
1173         rstart = bp->bio_offset;
1174         rend = bp->bio_offset + bp->bio_length;
1175         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1176                 if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1177                         continue;
1178                 for (i = 0; i < g_mirror_syncreqs; i++) {
1179                         sbp = disk->d_sync.ds_bios[i];
1180                         if (sbp == NULL)
1181                                 continue;
1182                         sstart = sbp->bio_offset;
1183                         send = sbp->bio_offset + sbp->bio_length;
1184                         if (rend > sstart && rstart < send)
1185                                 return (1);
1186                 }
1187         }
1188         return (0);
1189 }
1190
1191 /*
1192  * Return TRUE if the given sync request is colliding with a in-progress regular
1193  * request.
1194  */
1195 static int
1196 g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1197 {
1198         off_t rstart, rend, sstart, send;
1199         struct bio *bp;
1200
1201         if (sc->sc_sync.ds_ndisks == 0)
1202                 return (0);
1203         sstart = sbp->bio_offset;
1204         send = sbp->bio_offset + sbp->bio_length;
1205         TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1206                 rstart = bp->bio_offset;
1207                 rend = bp->bio_offset + bp->bio_length;
1208                 if (rend > sstart && rstart < send)
1209                         return (1);
1210         }
1211         return (0);
1212 }
1213
1214 /*
1215  * Puts request onto delayed queue.
1216  */
1217 static void
1218 g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1219 {
1220
1221         G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1222         bioq_insert_head(&sc->sc_regular_delayed, bp);
1223 }
1224
1225 /*
1226  * Puts synchronization request onto delayed queue.
1227  */
1228 static void
1229 g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1230 {
1231
1232         G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1233         bioq_insert_tail(&sc->sc_sync_delayed, bp);
1234 }
1235
1236 /*
1237  * Releases delayed regular requests which don't collide anymore with sync
1238  * requests.
1239  */
1240 static void
1241 g_mirror_regular_release(struct g_mirror_softc *sc)
1242 {
1243         struct bio *bp, *bp2;
1244
1245         TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1246                 if (g_mirror_sync_collision(sc, bp))
1247                         continue;
1248                 bioq_remove(&sc->sc_regular_delayed, bp);
1249                 G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1250                 mtx_lock(&sc->sc_queue_mtx);
1251                 bioq_insert_head(&sc->sc_queue, bp);
1252 #if 0
1253                 /*
1254                  * wakeup() is not needed, because this function is called from
1255                  * the worker thread.
1256                  */
1257                 wakeup(&sc->sc_queue);
1258 #endif
1259                 mtx_unlock(&sc->sc_queue_mtx);
1260         }
1261 }
1262
1263 /*
1264  * Releases delayed sync requests which don't collide anymore with regular
1265  * requests.
1266  */
1267 static void
1268 g_mirror_sync_release(struct g_mirror_softc *sc)
1269 {
1270         struct bio *bp, *bp2;
1271
1272         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1273                 if (g_mirror_regular_collision(sc, bp))
1274                         continue;
1275                 bioq_remove(&sc->sc_sync_delayed, bp);
1276                 G_MIRROR_LOGREQ(2, bp,
1277                     "Releasing delayed synchronization request.");
1278                 g_io_request(bp, bp->bio_from);
1279         }
1280 }
1281
1282 /*
1283  * Handle synchronization requests.
1284  * Every synchronization request is two-steps process: first, READ request is
1285  * send to active provider and then WRITE request (with read data) to the provider
1286  * beeing synchronized. When WRITE is finished, new synchronization request is
1287  * send.
1288  */
1289 static void
1290 g_mirror_sync_request(struct bio *bp)
1291 {
1292         struct g_mirror_softc *sc;
1293         struct g_mirror_disk *disk;
1294
1295         bp->bio_from->index--;
1296         sc = bp->bio_from->geom->softc;
1297         disk = bp->bio_from->private;
1298         if (disk == NULL) {
1299                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1300                 g_topology_lock();
1301                 g_mirror_kill_consumer(sc, bp->bio_from);
1302                 g_topology_unlock();
1303                 free(bp->bio_data, M_MIRROR);
1304                 g_destroy_bio(bp);
1305                 sx_xlock(&sc->sc_lock);
1306                 return;
1307         }
1308
1309         /*
1310          * Synchronization request.
1311          */
1312         switch (bp->bio_cmd) {
1313         case BIO_READ:
1314             {
1315                 struct g_consumer *cp;
1316
1317                 if (bp->bio_error != 0) {
1318                         G_MIRROR_LOGREQ(0, bp,
1319                             "Synchronization request failed (error=%d).",
1320                             bp->bio_error);
1321                         g_destroy_bio(bp);
1322                         return;
1323                 }
1324                 G_MIRROR_LOGREQ(3, bp,
1325                     "Synchronization request half-finished.");
1326                 bp->bio_cmd = BIO_WRITE;
1327                 bp->bio_cflags = 0;
1328                 cp = disk->d_consumer;
1329                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1330                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1331                     cp->acr, cp->acw, cp->ace));
1332                 cp->index++;
1333                 g_io_request(bp, cp);
1334                 return;
1335             }
1336         case BIO_WRITE:
1337             {
1338                 struct g_mirror_disk_sync *sync;
1339                 off_t offset;
1340                 void *data;
1341                 int i;
1342
1343                 if (bp->bio_error != 0) {
1344                         G_MIRROR_LOGREQ(0, bp,
1345                             "Synchronization request failed (error=%d).",
1346                             bp->bio_error);
1347                         g_destroy_bio(bp);
1348                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1349                         g_mirror_event_send(disk,
1350                             G_MIRROR_DISK_STATE_DISCONNECTED,
1351                             G_MIRROR_EVENT_DONTWAIT);
1352                         return;
1353                 }
1354                 G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1355                 sync = &disk->d_sync;
1356                 if (sync->ds_offset >= sc->sc_mediasize ||
1357                     sync->ds_consumer == NULL ||
1358                     (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1359                         /* Don't send more synchronization requests. */
1360                         sync->ds_inflight--;
1361                         if (sync->ds_bios != NULL) {
1362                                 i = (int)(uintptr_t)bp->bio_caller1;
1363                                 sync->ds_bios[i] = NULL;
1364                         }
1365                         free(bp->bio_data, M_MIRROR);
1366                         g_destroy_bio(bp);
1367                         if (sync->ds_inflight > 0)
1368                                 return;
1369                         if (sync->ds_consumer == NULL ||
1370                             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1371                                 return;
1372                         }
1373                         /* Disk up-to-date, activate it. */
1374                         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1375                             G_MIRROR_EVENT_DONTWAIT);
1376                         return;
1377                 }
1378
1379                 /* Send next synchronization request. */
1380                 data = bp->bio_data;
1381                 bzero(bp, sizeof(*bp));
1382                 bp->bio_cmd = BIO_READ;
1383                 bp->bio_offset = sync->ds_offset;
1384                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1385                 sync->ds_offset += bp->bio_length;
1386                 bp->bio_done = g_mirror_sync_done;
1387                 bp->bio_data = data;
1388                 bp->bio_from = sync->ds_consumer;
1389                 bp->bio_to = sc->sc_provider;
1390                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1391                 sync->ds_consumer->index++;
1392                 /*
1393                  * Delay the request if it is colliding with a regular request.
1394                  */
1395                 if (g_mirror_regular_collision(sc, bp))
1396                         g_mirror_sync_delay(sc, bp);
1397                 else
1398                         g_io_request(bp, sync->ds_consumer);
1399
1400                 /* Release delayed requests if possible. */
1401                 g_mirror_regular_release(sc);
1402
1403                 /* Find the smallest offset */
1404                 offset = sc->sc_mediasize;
1405                 for (i = 0; i < g_mirror_syncreqs; i++) {
1406                         bp = sync->ds_bios[i];
1407                         if (bp->bio_offset < offset)
1408                                 offset = bp->bio_offset;
1409                 }
1410                 if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
1411                         /* Update offset_done on every 100 blocks. */
1412                         sync->ds_offset_done = offset;
1413                         g_mirror_update_metadata(disk);
1414                 }
1415                 return;
1416             }
1417         default:
1418                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1419                     bp->bio_cmd, sc->sc_name));
1420                 break;
1421         }
1422 }
1423
1424 static void
1425 g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1426 {
1427         struct g_mirror_disk *disk;
1428         struct g_consumer *cp;
1429         struct bio *cbp;
1430
1431         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1432                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1433                         break;
1434         }
1435         if (disk == NULL) {
1436                 if (bp->bio_error == 0)
1437                         bp->bio_error = ENXIO;
1438                 g_io_deliver(bp, bp->bio_error);
1439                 return;
1440         }
1441         cbp = g_clone_bio(bp);
1442         if (cbp == NULL) {
1443                 if (bp->bio_error == 0)
1444                         bp->bio_error = ENOMEM;
1445                 g_io_deliver(bp, bp->bio_error);
1446                 return;
1447         }
1448         /*
1449          * Fill in the component buf structure.
1450          */
1451         cp = disk->d_consumer;
1452         cbp->bio_done = g_mirror_done;
1453         cbp->bio_to = cp->provider;
1454         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1455         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1456             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1457             cp->acw, cp->ace));
1458         cp->index++;
1459         g_io_request(cbp, cp);
1460 }
1461
1462 static void
1463 g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1464 {
1465         struct g_mirror_disk *disk;
1466         struct g_consumer *cp;
1467         struct bio *cbp;
1468
1469         disk = g_mirror_get_disk(sc);
1470         if (disk == NULL) {
1471                 if (bp->bio_error == 0)
1472                         bp->bio_error = ENXIO;
1473                 g_io_deliver(bp, bp->bio_error);
1474                 return;
1475         }
1476         cbp = g_clone_bio(bp);
1477         if (cbp == NULL) {
1478                 if (bp->bio_error == 0)
1479                         bp->bio_error = ENOMEM;
1480                 g_io_deliver(bp, bp->bio_error);
1481                 return;
1482         }
1483         /*
1484          * Fill in the component buf structure.
1485          */
1486         cp = disk->d_consumer;
1487         cbp->bio_done = g_mirror_done;
1488         cbp->bio_to = cp->provider;
1489         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1490         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1491             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1492             cp->acw, cp->ace));
1493         cp->index++;
1494         g_io_request(cbp, cp);
1495 }
1496
1497 #define TRACK_SIZE  (1 * 1024 * 1024)
1498 #define LOAD_SCALE      256
1499 #define ABS(x)          (((x) >= 0) ? (x) : (-(x)))
1500
1501 static void
1502 g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1503 {
1504         struct g_mirror_disk *disk, *dp;
1505         struct g_consumer *cp;
1506         struct bio *cbp;
1507         int prio, best;
1508
1509         /* Find a disk with the smallest load. */
1510         disk = NULL;
1511         best = INT_MAX;
1512         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1513                 if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1514                         continue;
1515                 prio = dp->load;
1516                 /* If disk head is precisely in position - highly prefer it. */
1517                 if (dp->d_last_offset == bp->bio_offset)
1518                         prio -= 2 * LOAD_SCALE;
1519                 else
1520                 /* If disk head is close to position - prefer it. */
1521                 if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1522                         prio -= 1 * LOAD_SCALE;
1523                 if (prio <= best) {
1524                         disk = dp;
1525                         best = prio;
1526                 }
1527         }
1528         KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1529         cbp = g_clone_bio(bp);
1530         if (cbp == NULL) {
1531                 if (bp->bio_error == 0)
1532                         bp->bio_error = ENOMEM;
1533                 g_io_deliver(bp, bp->bio_error);
1534                 return;
1535         }
1536         /*
1537          * Fill in the component buf structure.
1538          */
1539         cp = disk->d_consumer;
1540         cbp->bio_done = g_mirror_done;
1541         cbp->bio_to = cp->provider;
1542         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1543         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1544             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1545             cp->acw, cp->ace));
1546         cp->index++;
1547         /* Remember last head position */
1548         disk->d_last_offset = bp->bio_offset + bp->bio_length;
1549         /* Update loads. */
1550         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1551                 dp->load = (dp->d_consumer->index * LOAD_SCALE +
1552                     dp->load * 7) / 8;
1553         }
1554         g_io_request(cbp, cp);
1555 }
1556
1557 static void
1558 g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1559 {
1560         struct bio_queue_head queue;
1561         struct g_mirror_disk *disk;
1562         struct g_consumer *cp;
1563         struct bio *cbp;
1564         off_t left, mod, offset, slice;
1565         u_char *data;
1566         u_int ndisks;
1567
1568         if (bp->bio_length <= sc->sc_slice) {
1569                 g_mirror_request_round_robin(sc, bp);
1570                 return;
1571         }
1572         ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1573         slice = bp->bio_length / ndisks;
1574         mod = slice % sc->sc_provider->sectorsize;
1575         if (mod != 0)
1576                 slice += sc->sc_provider->sectorsize - mod;
1577         /*
1578          * Allocate all bios before sending any request, so we can
1579          * return ENOMEM in nice and clean way.
1580          */
1581         left = bp->bio_length;
1582         offset = bp->bio_offset;
1583         data = bp->bio_data;
1584         bioq_init(&queue);
1585         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1586                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1587                         continue;
1588                 cbp = g_clone_bio(bp);
1589                 if (cbp == NULL) {
1590                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1591                                 g_destroy_bio(cbp);
1592                         if (bp->bio_error == 0)
1593                                 bp->bio_error = ENOMEM;
1594                         g_io_deliver(bp, bp->bio_error);
1595                         return;
1596                 }
1597                 bioq_insert_tail(&queue, cbp);
1598                 cbp->bio_done = g_mirror_done;
1599                 cbp->bio_caller1 = disk;
1600                 cbp->bio_to = disk->d_consumer->provider;
1601                 cbp->bio_offset = offset;
1602                 cbp->bio_data = data;
1603                 cbp->bio_length = MIN(left, slice);
1604                 left -= cbp->bio_length;
1605                 if (left == 0)
1606                         break;
1607                 offset += cbp->bio_length;
1608                 data += cbp->bio_length;
1609         }
1610         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1611                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1612                 disk = cbp->bio_caller1;
1613                 cbp->bio_caller1 = NULL;
1614                 cp = disk->d_consumer;
1615                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1616                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1617                     cp->acr, cp->acw, cp->ace));
1618                 disk->d_consumer->index++;
1619                 g_io_request(cbp, disk->d_consumer);
1620         }
1621 }
1622
1623 static void
1624 g_mirror_register_request(struct bio *bp)
1625 {
1626         struct g_mirror_softc *sc;
1627
1628         sc = bp->bio_to->geom->softc;
1629         switch (bp->bio_cmd) {
1630         case BIO_READ:
1631                 switch (sc->sc_balance) {
1632                 case G_MIRROR_BALANCE_LOAD:
1633                         g_mirror_request_load(sc, bp);
1634                         break;
1635                 case G_MIRROR_BALANCE_PREFER:
1636                         g_mirror_request_prefer(sc, bp);
1637                         break;
1638                 case G_MIRROR_BALANCE_ROUND_ROBIN:
1639                         g_mirror_request_round_robin(sc, bp);
1640                         break;
1641                 case G_MIRROR_BALANCE_SPLIT:
1642                         g_mirror_request_split(sc, bp);
1643                         break;
1644                 }
1645                 return;
1646         case BIO_WRITE:
1647         case BIO_DELETE:
1648             {
1649                 struct g_mirror_disk *disk;
1650                 struct g_mirror_disk_sync *sync;
1651                 struct bio_queue_head queue;
1652                 struct g_consumer *cp;
1653                 struct bio *cbp;
1654
1655                 /*
1656                  * Delay the request if it is colliding with a synchronization
1657                  * request.
1658                  */
1659                 if (g_mirror_sync_collision(sc, bp)) {
1660                         g_mirror_regular_delay(sc, bp);
1661                         return;
1662                 }
1663
1664                 if (sc->sc_idle)
1665                         g_mirror_unidle(sc);
1666                 else
1667                         sc->sc_last_write = time_uptime;
1668
1669                 /*
1670                  * Allocate all bios before sending any request, so we can
1671                  * return ENOMEM in nice and clean way.
1672                  */
1673                 bioq_init(&queue);
1674                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1675                         sync = &disk->d_sync;
1676                         switch (disk->d_state) {
1677                         case G_MIRROR_DISK_STATE_ACTIVE:
1678                                 break;
1679                         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1680                                 if (bp->bio_offset >= sync->ds_offset)
1681                                         continue;
1682                                 break;
1683                         default:
1684                                 continue;
1685                         }
1686                         if (bp->bio_cmd == BIO_DELETE &&
1687                             (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1688                                 continue;
1689                         cbp = g_clone_bio(bp);
1690                         if (cbp == NULL) {
1691                                 while ((cbp = bioq_takefirst(&queue)) != NULL)
1692                                         g_destroy_bio(cbp);
1693                                 if (bp->bio_error == 0)
1694                                         bp->bio_error = ENOMEM;
1695                                 g_io_deliver(bp, bp->bio_error);
1696                                 return;
1697                         }
1698                         bioq_insert_tail(&queue, cbp);
1699                         cbp->bio_done = g_mirror_done;
1700                         cp = disk->d_consumer;
1701                         cbp->bio_caller1 = cp;
1702                         cbp->bio_to = cp->provider;
1703                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1704                             ("Consumer %s not opened (r%dw%de%d).",
1705                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1706                 }
1707                 if (bioq_first(&queue) == NULL) {
1708                         g_io_deliver(bp, EOPNOTSUPP);
1709                         return;
1710                 }
1711                 while ((cbp = bioq_takefirst(&queue)) != NULL) {
1712                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1713                         cp = cbp->bio_caller1;
1714                         cbp->bio_caller1 = NULL;
1715                         cp->index++;
1716                         sc->sc_writes++;
1717                         g_io_request(cbp, cp);
1718                 }
1719                 /*
1720                  * Put request onto inflight queue, so we can check if new
1721                  * synchronization requests don't collide with it.
1722                  */
1723                 bioq_insert_tail(&sc->sc_inflight, bp);
1724                 /*
1725                  * Bump syncid on first write.
1726                  */
1727                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1728                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1729                         g_mirror_bump_syncid(sc);
1730                 }
1731                 return;
1732             }
1733         default:
1734                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1735                     bp->bio_cmd, sc->sc_name));
1736                 break;
1737         }
1738 }
1739
1740 static int
1741 g_mirror_can_destroy(struct g_mirror_softc *sc)
1742 {
1743         struct g_geom *gp;
1744         struct g_consumer *cp;
1745
1746         g_topology_assert();
1747         gp = sc->sc_geom;
1748         if (gp->softc == NULL)
1749                 return (1);
1750         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1751                 return (0);
1752         LIST_FOREACH(cp, &gp->consumer, consumer) {
1753                 if (g_mirror_is_busy(sc, cp))
1754                         return (0);
1755         }
1756         gp = sc->sc_sync.ds_geom;
1757         LIST_FOREACH(cp, &gp->consumer, consumer) {
1758                 if (g_mirror_is_busy(sc, cp))
1759                         return (0);
1760         }
1761         G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1762             sc->sc_name);
1763         return (1);
1764 }
1765
1766 static int
1767 g_mirror_try_destroy(struct g_mirror_softc *sc)
1768 {
1769
1770         if (sc->sc_rootmount != NULL) {
1771                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1772                     sc->sc_rootmount);
1773                 root_mount_rel(sc->sc_rootmount);
1774                 sc->sc_rootmount = NULL;
1775         }
1776         g_topology_lock();
1777         if (!g_mirror_can_destroy(sc)) {
1778                 g_topology_unlock();
1779                 return (0);
1780         }
1781         sc->sc_geom->softc = NULL;
1782         sc->sc_sync.ds_geom->softc = NULL;
1783         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
1784                 g_topology_unlock();
1785                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1786                     &sc->sc_worker);
1787                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
1788                 sx_xunlock(&sc->sc_lock);
1789                 wakeup(&sc->sc_worker);
1790                 sc->sc_worker = NULL;
1791         } else {
1792                 g_topology_unlock();
1793                 g_mirror_destroy_device(sc);
1794                 free(sc, M_MIRROR);
1795         }
1796         return (1);
1797 }
1798
1799 /*
1800  * Worker thread.
1801  */
1802 static void
1803 g_mirror_worker(void *arg)
1804 {
1805         struct g_mirror_softc *sc;
1806         struct g_mirror_event *ep;
1807         struct bio *bp;
1808         int timeout;
1809
1810         sc = arg;
1811         thread_lock(curthread);
1812         sched_prio(curthread, PRIBIO);
1813         thread_unlock(curthread);
1814
1815         sx_xlock(&sc->sc_lock);
1816         for (;;) {
1817                 G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1818                 /*
1819                  * First take a look at events.
1820                  * This is important to handle events before any I/O requests.
1821                  */
1822                 ep = g_mirror_event_get(sc);
1823                 if (ep != NULL) {
1824                         g_mirror_event_remove(sc, ep);
1825                         if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1826                                 /* Update only device status. */
1827                                 G_MIRROR_DEBUG(3,
1828                                     "Running event for device %s.",
1829                                     sc->sc_name);
1830                                 ep->e_error = 0;
1831                                 g_mirror_update_device(sc, 1);
1832                         } else {
1833                                 /* Update disk status. */
1834                                 G_MIRROR_DEBUG(3, "Running event for disk %s.",
1835                                      g_mirror_get_diskname(ep->e_disk));
1836                                 ep->e_error = g_mirror_update_disk(ep->e_disk,
1837                                     ep->e_state);
1838                                 if (ep->e_error == 0)
1839                                         g_mirror_update_device(sc, 0);
1840                         }
1841                         if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1842                                 KASSERT(ep->e_error == 0,
1843                                     ("Error cannot be handled."));
1844                                 g_mirror_event_free(ep);
1845                         } else {
1846                                 ep->e_flags |= G_MIRROR_EVENT_DONE;
1847                                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1848                                     ep);
1849                                 mtx_lock(&sc->sc_events_mtx);
1850                                 wakeup(ep);
1851                                 mtx_unlock(&sc->sc_events_mtx);
1852                         }
1853                         if ((sc->sc_flags &
1854                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1855                                 if (g_mirror_try_destroy(sc)) {
1856                                         curthread->td_pflags &= ~TDP_GEOM;
1857                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1858                                         kproc_exit(0);
1859                                 }
1860                         }
1861                         G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1862                         continue;
1863                 }
1864                 /*
1865                  * Check if we can mark array as CLEAN and if we can't take
1866                  * how much seconds should we wait.
1867                  */
1868                 timeout = g_mirror_idle(sc, -1);
1869                 /*
1870                  * Now I/O requests.
1871                  */
1872                 /* Get first request from the queue. */
1873                 mtx_lock(&sc->sc_queue_mtx);
1874                 bp = bioq_takefirst(&sc->sc_queue);
1875                 if (bp == NULL) {
1876                         if ((sc->sc_flags &
1877                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1878                                 mtx_unlock(&sc->sc_queue_mtx);
1879                                 if (g_mirror_try_destroy(sc)) {
1880                                         curthread->td_pflags &= ~TDP_GEOM;
1881                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1882                                         kproc_exit(0);
1883                                 }
1884                                 mtx_lock(&sc->sc_queue_mtx);
1885                         }
1886                         sx_xunlock(&sc->sc_lock);
1887                         /*
1888                          * XXX: We can miss an event here, because an event
1889                          *      can be added without sx-device-lock and without
1890                          *      mtx-queue-lock. Maybe I should just stop using
1891                          *      dedicated mutex for events synchronization and
1892                          *      stick with the queue lock?
1893                          *      The event will hang here until next I/O request
1894                          *      or next event is received.
1895                          */
1896                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1897                             timeout * hz);
1898                         sx_xlock(&sc->sc_lock);
1899                         G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1900                         continue;
1901                 }
1902                 mtx_unlock(&sc->sc_queue_mtx);
1903
1904                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
1905                     (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
1906                         g_mirror_sync_request(bp);      /* READ */
1907                 } else if (bp->bio_to != sc->sc_provider) {
1908                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
1909                                 g_mirror_regular_request(bp);
1910                         else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
1911                                 g_mirror_sync_request(bp);      /* WRITE */
1912                         else {
1913                                 KASSERT(0,
1914                                     ("Invalid request cflags=0x%hhx to=%s.",
1915                                     bp->bio_cflags, bp->bio_to->name));
1916                         }
1917                 } else {
1918                         g_mirror_register_request(bp);
1919                 }
1920                 G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
1921         }
1922 }
1923
1924 static void
1925 g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
1926 {
1927
1928         sx_assert(&sc->sc_lock, SX_LOCKED);
1929
1930         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
1931                 return;
1932         if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
1933                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
1934                     g_mirror_get_diskname(disk), sc->sc_name);
1935                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1936         } else if (sc->sc_idle &&
1937             (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
1938                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
1939                     g_mirror_get_diskname(disk), sc->sc_name);
1940                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
1941         }
1942 }
1943
1944 static void
1945 g_mirror_sync_start(struct g_mirror_disk *disk)
1946 {
1947         struct g_mirror_softc *sc;
1948         struct g_consumer *cp;
1949         struct bio *bp;
1950         int error, i;
1951
1952         g_topology_assert_not();
1953         sc = disk->d_softc;
1954         sx_assert(&sc->sc_lock, SX_LOCKED);
1955
1956         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
1957             ("Disk %s is not marked for synchronization.",
1958             g_mirror_get_diskname(disk)));
1959         KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1960             ("Device not in RUNNING state (%s, %u).", sc->sc_name,
1961             sc->sc_state));
1962
1963         sx_xunlock(&sc->sc_lock);
1964         g_topology_lock();
1965         cp = g_new_consumer(sc->sc_sync.ds_geom);
1966         cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
1967         error = g_attach(cp, sc->sc_provider);
1968         KASSERT(error == 0,
1969             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
1970         error = g_access(cp, 1, 0, 0);
1971         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
1972         g_topology_unlock();
1973         sx_xlock(&sc->sc_lock);
1974
1975         G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
1976             g_mirror_get_diskname(disk));
1977         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
1978                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1979         KASSERT(disk->d_sync.ds_consumer == NULL,
1980             ("Sync consumer already exists (device=%s, disk=%s).",
1981             sc->sc_name, g_mirror_get_diskname(disk)));
1982
1983         disk->d_sync.ds_consumer = cp;
1984         disk->d_sync.ds_consumer->private = disk;
1985         disk->d_sync.ds_consumer->index = 0;
1986
1987         /*
1988          * Allocate memory for synchronization bios and initialize them.
1989          */
1990         disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
1991             M_MIRROR, M_WAITOK);
1992         for (i = 0; i < g_mirror_syncreqs; i++) {
1993                 bp = g_alloc_bio();
1994                 disk->d_sync.ds_bios[i] = bp;
1995                 bp->bio_parent = NULL;
1996                 bp->bio_cmd = BIO_READ;
1997                 bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
1998                 bp->bio_cflags = 0;
1999                 bp->bio_offset = disk->d_sync.ds_offset;
2000                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
2001                 disk->d_sync.ds_offset += bp->bio_length;
2002                 bp->bio_done = g_mirror_sync_done;
2003                 bp->bio_from = disk->d_sync.ds_consumer;
2004                 bp->bio_to = sc->sc_provider;
2005                 bp->bio_caller1 = (void *)(uintptr_t)i;
2006         }
2007
2008         /* Increase the number of disks in SYNCHRONIZING state. */
2009         sc->sc_sync.ds_ndisks++;
2010         /* Set the number of in-flight synchronization requests. */
2011         disk->d_sync.ds_inflight = g_mirror_syncreqs;
2012
2013         /*
2014          * Fire off first synchronization requests.
2015          */
2016         for (i = 0; i < g_mirror_syncreqs; i++) {
2017                 bp = disk->d_sync.ds_bios[i];
2018                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
2019                 disk->d_sync.ds_consumer->index++;
2020                 /*
2021                  * Delay the request if it is colliding with a regular request.
2022                  */
2023                 if (g_mirror_regular_collision(sc, bp))
2024                         g_mirror_sync_delay(sc, bp);
2025                 else
2026                         g_io_request(bp, disk->d_sync.ds_consumer);
2027         }
2028 }
2029
2030 /*
2031  * Stop synchronization process.
2032  * type: 0 - synchronization finished
2033  *       1 - synchronization stopped
2034  */
2035 static void
2036 g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2037 {
2038         struct g_mirror_softc *sc;
2039         struct g_consumer *cp;
2040
2041         g_topology_assert_not();
2042         sc = disk->d_softc;
2043         sx_assert(&sc->sc_lock, SX_LOCKED);
2044
2045         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2046             ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2047             g_mirror_disk_state2str(disk->d_state)));
2048         if (disk->d_sync.ds_consumer == NULL)
2049                 return;
2050
2051         if (type == 0) {
2052                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2053                     sc->sc_name, g_mirror_get_diskname(disk));
2054         } else /* if (type == 1) */ {
2055                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2056                     sc->sc_name, g_mirror_get_diskname(disk));
2057         }
2058         free(disk->d_sync.ds_bios, M_MIRROR);
2059         disk->d_sync.ds_bios = NULL;
2060         cp = disk->d_sync.ds_consumer;
2061         disk->d_sync.ds_consumer = NULL;
2062         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2063         sc->sc_sync.ds_ndisks--;
2064         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2065         g_topology_lock();
2066         g_mirror_kill_consumer(sc, cp);
2067         g_topology_unlock();
2068         sx_xlock(&sc->sc_lock);
2069 }
2070
2071 static void
2072 g_mirror_launch_provider(struct g_mirror_softc *sc)
2073 {
2074         struct g_mirror_disk *disk;
2075         struct g_provider *pp, *dp;
2076
2077         sx_assert(&sc->sc_lock, SX_LOCKED);
2078
2079         g_topology_lock();
2080         pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2081         pp->flags |= G_PF_DIRECT_RECEIVE;
2082         pp->mediasize = sc->sc_mediasize;
2083         pp->sectorsize = sc->sc_sectorsize;
2084         pp->stripesize = 0;
2085         pp->stripeoffset = 0;
2086
2087         /* Splitting of unmapped BIO's could work but isn't implemented now */
2088         if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2089                 pp->flags |= G_PF_ACCEPT_UNMAPPED;
2090
2091         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2092                 if (disk->d_consumer && disk->d_consumer->provider) {
2093                         dp = disk->d_consumer->provider;
2094                         if (dp->stripesize > pp->stripesize) {
2095                                 pp->stripesize = dp->stripesize;
2096                                 pp->stripeoffset = dp->stripeoffset;
2097                         }
2098                         /* A provider underneath us doesn't support unmapped */
2099                         if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2100                                 G_MIRROR_DEBUG(0, "Cancelling unmapped "
2101                                     "because of %s.", dp->name);
2102                                 pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2103                         }
2104                 }
2105         }
2106         sc->sc_provider = pp;
2107         g_error_provider(pp, 0);
2108         g_topology_unlock();
2109         G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2110             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2111         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2112                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2113                         g_mirror_sync_start(disk);
2114         }
2115 }
2116
2117 static void
2118 g_mirror_destroy_provider(struct g_mirror_softc *sc)
2119 {
2120         struct g_mirror_disk *disk;
2121         struct bio *bp;
2122
2123         g_topology_assert_not();
2124         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2125             sc->sc_name));
2126
2127         g_topology_lock();
2128         g_error_provider(sc->sc_provider, ENXIO);
2129         mtx_lock(&sc->sc_queue_mtx);
2130         while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL)
2131                 g_io_deliver(bp, ENXIO);
2132         mtx_unlock(&sc->sc_queue_mtx);
2133         G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2134             sc->sc_provider->name);
2135         sc->sc_provider->flags |= G_PF_WITHER;
2136         g_orphan_provider(sc->sc_provider, ENXIO);
2137         g_topology_unlock();
2138         sc->sc_provider = NULL;
2139         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2140                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2141                         g_mirror_sync_stop(disk, 1);
2142         }
2143 }
2144
2145 static void
2146 g_mirror_go(void *arg)
2147 {
2148         struct g_mirror_softc *sc;
2149
2150         sc = arg;
2151         G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2152         g_mirror_event_send(sc, 0,
2153             G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2154 }
2155
2156 static u_int
2157 g_mirror_determine_state(struct g_mirror_disk *disk)
2158 {
2159         struct g_mirror_softc *sc;
2160         u_int state;
2161
2162         sc = disk->d_softc;
2163         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2164                 if ((disk->d_flags &
2165                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2166                         /* Disk does not need synchronization. */
2167                         state = G_MIRROR_DISK_STATE_ACTIVE;
2168                 } else {
2169                         if ((sc->sc_flags &
2170                              G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2171                             (disk->d_flags &
2172                              G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2173                                 /*
2174                                  * We can start synchronization from
2175                                  * the stored offset.
2176                                  */
2177                                 state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2178                         } else {
2179                                 state = G_MIRROR_DISK_STATE_STALE;
2180                         }
2181                 }
2182         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2183                 /*
2184                  * Reset all synchronization data for this disk,
2185                  * because if it even was synchronized, it was
2186                  * synchronized to disks with different syncid.
2187                  */
2188                 disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2189                 disk->d_sync.ds_offset = 0;
2190                 disk->d_sync.ds_offset_done = 0;
2191                 disk->d_sync.ds_syncid = sc->sc_syncid;
2192                 if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2193                     (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2194                         state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2195                 } else {
2196                         state = G_MIRROR_DISK_STATE_STALE;
2197                 }
2198         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2199                 /*
2200                  * Not good, NOT GOOD!
2201                  * It means that mirror was started on stale disks
2202                  * and more fresh disk just arrive.
2203                  * If there were writes, mirror is broken, sorry.
2204                  * I think the best choice here is don't touch
2205                  * this disk and inform the user loudly.
2206                  */
2207                 G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2208                     "disk (%s) arrives!! It will not be connected to the "
2209                     "running device.", sc->sc_name,
2210                     g_mirror_get_diskname(disk));
2211                 g_mirror_destroy_disk(disk);
2212                 state = G_MIRROR_DISK_STATE_NONE;
2213                 /* Return immediately, because disk was destroyed. */
2214                 return (state);
2215         }
2216         G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2217             g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2218         return (state);
2219 }
2220
2221 /*
2222  * Update device state.
2223  */
2224 static void
2225 g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
2226 {
2227         struct g_mirror_disk *disk;
2228         u_int state;
2229
2230         sx_assert(&sc->sc_lock, SX_XLOCKED);
2231
2232         switch (sc->sc_state) {
2233         case G_MIRROR_DEVICE_STATE_STARTING:
2234             {
2235                 struct g_mirror_disk *pdisk, *tdisk;
2236                 u_int dirty, ndisks, genid, syncid;
2237
2238                 KASSERT(sc->sc_provider == NULL,
2239                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2240                 /*
2241                  * Are we ready? We are, if all disks are connected or
2242                  * if we have any disks and 'force' is true.
2243                  */
2244                 ndisks = g_mirror_ndisks(sc, -1);
2245                 if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
2246                         ;
2247                 } else if (ndisks == 0) {
2248                         /*
2249                          * Disks went down in starting phase, so destroy
2250                          * device.
2251                          */
2252                         callout_drain(&sc->sc_callout);
2253                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2254                         G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2255                             sc->sc_rootmount);
2256                         root_mount_rel(sc->sc_rootmount);
2257                         sc->sc_rootmount = NULL;
2258                         return;
2259                 } else {
2260                         return;
2261                 }
2262
2263                 /*
2264                  * Activate all disks with the biggest syncid.
2265                  */
2266                 if (force) {
2267                         /*
2268                          * If 'force' is true, we have been called due to
2269                          * timeout, so don't bother canceling timeout.
2270                          */
2271                         ndisks = 0;
2272                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2273                                 if ((disk->d_flags &
2274                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2275                                         ndisks++;
2276                                 }
2277                         }
2278                         if (ndisks == 0) {
2279                                 /* No valid disks found, destroy device. */
2280                                 sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2281                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2282                                     __LINE__, sc->sc_rootmount);
2283                                 root_mount_rel(sc->sc_rootmount);
2284                                 sc->sc_rootmount = NULL;
2285                                 return;
2286                         }
2287                 } else {
2288                         /* Cancel timeout. */
2289                         callout_drain(&sc->sc_callout);
2290                 }
2291
2292                 /*
2293                  * Find the biggest genid.
2294                  */
2295                 genid = 0;
2296                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2297                         if (disk->d_genid > genid)
2298                                 genid = disk->d_genid;
2299                 }
2300                 sc->sc_genid = genid;
2301                 /*
2302                  * Remove all disks without the biggest genid.
2303                  */
2304                 LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2305                         if (disk->d_genid < genid) {
2306                                 G_MIRROR_DEBUG(0,
2307                                     "Component %s (device %s) broken, skipping.",
2308                                     g_mirror_get_diskname(disk), sc->sc_name);
2309                                 g_mirror_destroy_disk(disk);
2310                         }
2311                 }
2312
2313                 /*
2314                  * Find the biggest syncid.
2315                  */
2316                 syncid = 0;
2317                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2318                         if (disk->d_sync.ds_syncid > syncid)
2319                                 syncid = disk->d_sync.ds_syncid;
2320                 }
2321
2322                 /*
2323                  * Here we need to look for dirty disks and if all disks
2324                  * with the biggest syncid are dirty, we have to choose
2325                  * one with the biggest priority and rebuild the rest.
2326                  */
2327                 /*
2328                  * Find the number of dirty disks with the biggest syncid.
2329                  * Find the number of disks with the biggest syncid.
2330                  * While here, find a disk with the biggest priority.
2331                  */
2332                 dirty = ndisks = 0;
2333                 pdisk = NULL;
2334                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2335                         if (disk->d_sync.ds_syncid != syncid)
2336                                 continue;
2337                         if ((disk->d_flags &
2338                             G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2339                                 continue;
2340                         }
2341                         ndisks++;
2342                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2343                                 dirty++;
2344                                 if (pdisk == NULL ||
2345                                     pdisk->d_priority < disk->d_priority) {
2346                                         pdisk = disk;
2347                                 }
2348                         }
2349                 }
2350                 if (dirty == 0) {
2351                         /* No dirty disks at all, great. */
2352                 } else if (dirty == ndisks) {
2353                         /*
2354                          * Force synchronization for all dirty disks except one
2355                          * with the biggest priority.
2356                          */
2357                         KASSERT(pdisk != NULL, ("pdisk == NULL"));
2358                         G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2359                             "master disk for synchronization.",
2360                             g_mirror_get_diskname(pdisk), sc->sc_name);
2361                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2362                                 if (disk->d_sync.ds_syncid != syncid)
2363                                         continue;
2364                                 if ((disk->d_flags &
2365                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2366                                         continue;
2367                                 }
2368                                 KASSERT((disk->d_flags &
2369                                     G_MIRROR_DISK_FLAG_DIRTY) != 0,
2370                                     ("Disk %s isn't marked as dirty.",
2371                                     g_mirror_get_diskname(disk)));
2372                                 /* Skip the disk with the biggest priority. */
2373                                 if (disk == pdisk)
2374                                         continue;
2375                                 disk->d_sync.ds_syncid = 0;
2376                         }
2377                 } else if (dirty < ndisks) {
2378                         /*
2379                          * Force synchronization for all dirty disks.
2380                          * We have some non-dirty disks.
2381                          */
2382                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2383                                 if (disk->d_sync.ds_syncid != syncid)
2384                                         continue;
2385                                 if ((disk->d_flags &
2386                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2387                                         continue;
2388                                 }
2389                                 if ((disk->d_flags &
2390                                     G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2391                                         continue;
2392                                 }
2393                                 disk->d_sync.ds_syncid = 0;
2394                         }
2395                 }
2396
2397                 /* Reset hint. */
2398                 sc->sc_hint = NULL;
2399                 sc->sc_syncid = syncid;
2400                 if (force) {
2401                         /* Remember to bump syncid on first write. */
2402                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2403                 }
2404                 state = G_MIRROR_DEVICE_STATE_RUNNING;
2405                 G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2406                     sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2407                     g_mirror_device_state2str(state));
2408                 sc->sc_state = state;
2409                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2410                         state = g_mirror_determine_state(disk);
2411                         g_mirror_event_send(disk, state,
2412                             G_MIRROR_EVENT_DONTWAIT);
2413                         if (state == G_MIRROR_DISK_STATE_STALE)
2414                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2415                 }
2416                 break;
2417             }
2418         case G_MIRROR_DEVICE_STATE_RUNNING:
2419                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2420                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2421                         /*
2422                          * No active disks or no disks at all,
2423                          * so destroy device.
2424                          */
2425                         if (sc->sc_provider != NULL)
2426                                 g_mirror_destroy_provider(sc);
2427                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2428                         break;
2429                 } else if (g_mirror_ndisks(sc,
2430                     G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2431                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2432                         /*
2433                          * We have active disks, launch provider if it doesn't
2434                          * exist.
2435                          */
2436                         if (sc->sc_provider == NULL)
2437                                 g_mirror_launch_provider(sc);
2438                         if (sc->sc_rootmount != NULL) {
2439                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2440                                     __LINE__, sc->sc_rootmount);
2441                                 root_mount_rel(sc->sc_rootmount);
2442                                 sc->sc_rootmount = NULL;
2443                         }
2444                 }
2445                 /*
2446                  * Genid should be bumped immediately, so do it here.
2447                  */
2448                 if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2449                         sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2450                         g_mirror_bump_genid(sc);
2451                 }
2452                 break;
2453         default:
2454                 KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2455                     sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2456                 break;
2457         }
2458 }
2459
2460 /*
2461  * Update disk state and device state if needed.
2462  */
2463 #define DISK_STATE_CHANGED()    G_MIRROR_DEBUG(1,                       \
2464         "Disk %s state changed from %s to %s (device %s).",             \
2465         g_mirror_get_diskname(disk),                                    \
2466         g_mirror_disk_state2str(disk->d_state),                         \
2467         g_mirror_disk_state2str(state), sc->sc_name)
2468 static int
2469 g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2470 {
2471         struct g_mirror_softc *sc;
2472
2473         sc = disk->d_softc;
2474         sx_assert(&sc->sc_lock, SX_XLOCKED);
2475
2476 again:
2477         G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2478             g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2479             g_mirror_disk_state2str(state));
2480         switch (state) {
2481         case G_MIRROR_DISK_STATE_NEW:
2482                 /*
2483                  * Possible scenarios:
2484                  * 1. New disk arrive.
2485                  */
2486                 /* Previous state should be NONE. */
2487                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2488                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2489                     g_mirror_disk_state2str(disk->d_state)));
2490                 DISK_STATE_CHANGED();
2491
2492                 disk->d_state = state;
2493                 if (LIST_EMPTY(&sc->sc_disks))
2494                         LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2495                 else {
2496                         struct g_mirror_disk *dp;
2497
2498                         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2499                                 if (disk->d_priority >= dp->d_priority) {
2500                                         LIST_INSERT_BEFORE(dp, disk, d_next);
2501                                         dp = NULL;
2502                                         break;
2503                                 }
2504                                 if (LIST_NEXT(dp, d_next) == NULL)
2505                                         break;
2506                         }
2507                         if (dp != NULL)
2508                                 LIST_INSERT_AFTER(dp, disk, d_next);
2509                 }
2510                 G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2511                     sc->sc_name, g_mirror_get_diskname(disk));
2512                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2513                         break;
2514                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2515                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2516                     g_mirror_device_state2str(sc->sc_state),
2517                     g_mirror_get_diskname(disk),
2518                     g_mirror_disk_state2str(disk->d_state)));
2519                 state = g_mirror_determine_state(disk);
2520                 if (state != G_MIRROR_DISK_STATE_NONE)
2521                         goto again;
2522                 break;
2523         case G_MIRROR_DISK_STATE_ACTIVE:
2524                 /*
2525                  * Possible scenarios:
2526                  * 1. New disk does not need synchronization.
2527                  * 2. Synchronization process finished successfully.
2528                  */
2529                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2530                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2531                     g_mirror_device_state2str(sc->sc_state),
2532                     g_mirror_get_diskname(disk),
2533                     g_mirror_disk_state2str(disk->d_state)));
2534                 /* Previous state should be NEW or SYNCHRONIZING. */
2535                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2536                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2537                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2538                     g_mirror_disk_state2str(disk->d_state)));
2539                 DISK_STATE_CHANGED();
2540
2541                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2542                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2543                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2544                         g_mirror_sync_stop(disk, 0);
2545                 }
2546                 disk->d_state = state;
2547                 disk->d_sync.ds_offset = 0;
2548                 disk->d_sync.ds_offset_done = 0;
2549                 g_mirror_update_idle(sc, disk);
2550                 g_mirror_update_metadata(disk);
2551                 G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2552                     sc->sc_name, g_mirror_get_diskname(disk));
2553                 break;
2554         case G_MIRROR_DISK_STATE_STALE:
2555                 /*
2556                  * Possible scenarios:
2557                  * 1. Stale disk was connected.
2558                  */
2559                 /* Previous state should be NEW. */
2560                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2561                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2562                     g_mirror_disk_state2str(disk->d_state)));
2563                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2564                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2565                     g_mirror_device_state2str(sc->sc_state),
2566                     g_mirror_get_diskname(disk),
2567                     g_mirror_disk_state2str(disk->d_state)));
2568                 /*
2569                  * STALE state is only possible if device is marked
2570                  * NOAUTOSYNC.
2571                  */
2572                 KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2573                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2574                     g_mirror_device_state2str(sc->sc_state),
2575                     g_mirror_get_diskname(disk),
2576                     g_mirror_disk_state2str(disk->d_state)));
2577                 DISK_STATE_CHANGED();
2578
2579                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2580                 disk->d_state = state;
2581                 g_mirror_update_metadata(disk);
2582                 G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2583                     sc->sc_name, g_mirror_get_diskname(disk));
2584                 break;
2585         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2586                 /*
2587                  * Possible scenarios:
2588                  * 1. Disk which needs synchronization was connected.
2589                  */
2590                 /* Previous state should be NEW. */
2591                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2592                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2593                     g_mirror_disk_state2str(disk->d_state)));
2594                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2595                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2596                     g_mirror_device_state2str(sc->sc_state),
2597                     g_mirror_get_diskname(disk),
2598                     g_mirror_disk_state2str(disk->d_state)));
2599                 DISK_STATE_CHANGED();
2600
2601                 if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2602                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2603                 disk->d_state = state;
2604                 if (sc->sc_provider != NULL) {
2605                         g_mirror_sync_start(disk);
2606                         g_mirror_update_metadata(disk);
2607                 }
2608                 break;
2609         case G_MIRROR_DISK_STATE_DISCONNECTED:
2610                 /*
2611                  * Possible scenarios:
2612                  * 1. Device wasn't running yet, but disk disappear.
2613                  * 2. Disk was active and disapppear.
2614                  * 3. Disk disappear during synchronization process.
2615                  */
2616                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2617                         /*
2618                          * Previous state should be ACTIVE, STALE or
2619                          * SYNCHRONIZING.
2620                          */
2621                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2622                             disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2623                             disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2624                             ("Wrong disk state (%s, %s).",
2625                             g_mirror_get_diskname(disk),
2626                             g_mirror_disk_state2str(disk->d_state)));
2627                 } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2628                         /* Previous state should be NEW. */
2629                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2630                             ("Wrong disk state (%s, %s).",
2631                             g_mirror_get_diskname(disk),
2632                             g_mirror_disk_state2str(disk->d_state)));
2633                         /*
2634                          * Reset bumping syncid if disk disappeared in STARTING
2635                          * state.
2636                          */
2637                         if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2638                                 sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2639 #ifdef  INVARIANTS
2640                 } else {
2641                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2642                             sc->sc_name,
2643                             g_mirror_device_state2str(sc->sc_state),
2644                             g_mirror_get_diskname(disk),
2645                             g_mirror_disk_state2str(disk->d_state)));
2646 #endif
2647                 }
2648                 DISK_STATE_CHANGED();
2649                 G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2650                     sc->sc_name, g_mirror_get_diskname(disk));
2651
2652                 g_mirror_destroy_disk(disk);
2653                 break;
2654         case G_MIRROR_DISK_STATE_DESTROY:
2655             {
2656                 int error;
2657
2658                 error = g_mirror_clear_metadata(disk);
2659                 if (error != 0)
2660                         return (error);
2661                 DISK_STATE_CHANGED();
2662                 G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2663                     sc->sc_name, g_mirror_get_diskname(disk));
2664
2665                 g_mirror_destroy_disk(disk);
2666                 sc->sc_ndisks--;
2667                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2668                         g_mirror_update_metadata(disk);
2669                 }
2670                 break;
2671             }
2672         default:
2673                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2674                 break;
2675         }
2676         return (0);
2677 }
2678 #undef  DISK_STATE_CHANGED
2679
2680 int
2681 g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2682 {
2683         struct g_provider *pp;
2684         u_char *buf;
2685         int error;
2686
2687         g_topology_assert();
2688
2689         error = g_access(cp, 1, 0, 0);
2690         if (error != 0)
2691                 return (error);
2692         pp = cp->provider;
2693         g_topology_unlock();
2694         /* Metadata are stored on last sector. */
2695         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2696             &error);
2697         g_topology_lock();
2698         g_access(cp, -1, 0, 0);
2699         if (buf == NULL) {
2700                 G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2701                     cp->provider->name, error);
2702                 return (error);
2703         }
2704
2705         /* Decode metadata. */
2706         error = mirror_metadata_decode(buf, md);
2707         g_free(buf);
2708         if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2709                 return (EINVAL);
2710         if (md->md_version > G_MIRROR_VERSION) {
2711                 G_MIRROR_DEBUG(0,
2712                     "Kernel module is too old to handle metadata from %s.",
2713                     cp->provider->name);
2714                 return (EINVAL);
2715         }
2716         if (error != 0) {
2717                 G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2718                     cp->provider->name);
2719                 return (error);
2720         }
2721
2722         return (0);
2723 }
2724
2725 static int
2726 g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2727     struct g_mirror_metadata *md)
2728 {
2729
2730         if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2731                 G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2732                     pp->name, md->md_did);
2733                 return (EEXIST);
2734         }
2735         if (md->md_all != sc->sc_ndisks) {
2736                 G_MIRROR_DEBUG(1,
2737                     "Invalid '%s' field on disk %s (device %s), skipping.",
2738                     "md_all", pp->name, sc->sc_name);
2739                 return (EINVAL);
2740         }
2741         if (md->md_slice != sc->sc_slice) {
2742                 G_MIRROR_DEBUG(1,
2743                     "Invalid '%s' field on disk %s (device %s), skipping.",
2744                     "md_slice", pp->name, sc->sc_name);
2745                 return (EINVAL);
2746         }
2747         if (md->md_balance != sc->sc_balance) {
2748                 G_MIRROR_DEBUG(1,
2749                     "Invalid '%s' field on disk %s (device %s), skipping.",
2750                     "md_balance", pp->name, sc->sc_name);
2751                 return (EINVAL);
2752         }
2753 #if 0
2754         if (md->md_mediasize != sc->sc_mediasize) {
2755                 G_MIRROR_DEBUG(1,
2756                     "Invalid '%s' field on disk %s (device %s), skipping.",
2757                     "md_mediasize", pp->name, sc->sc_name);
2758                 return (EINVAL);
2759         }
2760 #endif
2761         if (sc->sc_mediasize > pp->mediasize) {
2762                 G_MIRROR_DEBUG(1,
2763                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2764                     sc->sc_name);
2765                 return (EINVAL);
2766         }
2767         if (md->md_sectorsize != sc->sc_sectorsize) {
2768                 G_MIRROR_DEBUG(1,
2769                     "Invalid '%s' field on disk %s (device %s), skipping.",
2770                     "md_sectorsize", pp->name, sc->sc_name);
2771                 return (EINVAL);
2772         }
2773         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2774                 G_MIRROR_DEBUG(1,
2775                     "Invalid sector size of disk %s (device %s), skipping.",
2776                     pp->name, sc->sc_name);
2777                 return (EINVAL);
2778         }
2779         if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2780                 G_MIRROR_DEBUG(1,
2781                     "Invalid device flags on disk %s (device %s), skipping.",
2782                     pp->name, sc->sc_name);
2783                 return (EINVAL);
2784         }
2785         if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2786                 G_MIRROR_DEBUG(1,
2787                     "Invalid disk flags on disk %s (device %s), skipping.",
2788                     pp->name, sc->sc_name);
2789                 return (EINVAL);
2790         }
2791         return (0);
2792 }
2793
2794 int
2795 g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2796     struct g_mirror_metadata *md)
2797 {
2798         struct g_mirror_disk *disk;
2799         int error;
2800
2801         g_topology_assert_not();
2802         G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2803
2804         error = g_mirror_check_metadata(sc, pp, md);
2805         if (error != 0)
2806                 return (error);
2807         if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
2808             md->md_genid < sc->sc_genid) {
2809                 G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2810                     pp->name, sc->sc_name);
2811                 return (EINVAL);
2812         }
2813         disk = g_mirror_init_disk(sc, pp, md, &error);
2814         if (disk == NULL)
2815                 return (error);
2816         error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2817             G_MIRROR_EVENT_WAIT);
2818         if (error != 0)
2819                 return (error);
2820         if (md->md_version < G_MIRROR_VERSION) {
2821                 G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2822                     pp->name, md->md_version, G_MIRROR_VERSION);
2823                 g_mirror_update_metadata(disk);
2824         }
2825         return (0);
2826 }
2827
2828 static void
2829 g_mirror_destroy_delayed(void *arg, int flag)
2830 {
2831         struct g_mirror_softc *sc;
2832         int error;
2833
2834         if (flag == EV_CANCEL) {
2835                 G_MIRROR_DEBUG(1, "Destroying canceled.");
2836                 return;
2837         }
2838         sc = arg;
2839         g_topology_unlock();
2840         sx_xlock(&sc->sc_lock);
2841         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
2842             ("DESTROY flag set on %s.", sc->sc_name));
2843         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
2844             ("DESTROYING flag not set on %s.", sc->sc_name));
2845         G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
2846         error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
2847         if (error != 0) {
2848                 G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
2849                     sc->sc_name, error);
2850                 sx_xunlock(&sc->sc_lock);
2851         }
2852         g_topology_lock();
2853 }
2854
2855 static int
2856 g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
2857 {
2858         struct g_mirror_softc *sc;
2859         int dcr, dcw, dce, error = 0;
2860
2861         g_topology_assert();
2862         G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
2863             acw, ace);
2864
2865         sc = pp->geom->softc;
2866         if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
2867                 return (0);
2868         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
2869
2870         dcr = pp->acr + acr;
2871         dcw = pp->acw + acw;
2872         dce = pp->ace + ace;
2873
2874         g_topology_unlock();
2875         sx_xlock(&sc->sc_lock);
2876         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
2877             LIST_EMPTY(&sc->sc_disks)) {
2878                 if (acr > 0 || acw > 0 || ace > 0)
2879                         error = ENXIO;
2880                 goto end;
2881         }
2882         if (dcw == 0)
2883                 g_mirror_idle(sc, dcw);
2884         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
2885                 if (acr > 0 || acw > 0 || ace > 0) {
2886                         error = ENXIO;
2887                         goto end;
2888                 }
2889                 if (dcr == 0 && dcw == 0 && dce == 0) {
2890                         g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
2891                             sc, NULL);
2892                 }
2893         }
2894 end:
2895         sx_xunlock(&sc->sc_lock);
2896         g_topology_lock();
2897         return (error);
2898 }
2899
2900 static struct g_geom *
2901 g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
2902 {
2903         struct g_mirror_softc *sc;
2904         struct g_geom *gp;
2905         int error, timeout;
2906
2907         g_topology_assert();
2908         G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
2909             md->md_mid);
2910
2911         /* One disk is minimum. */
2912         if (md->md_all < 1)
2913                 return (NULL);
2914         /*
2915          * Action geom.
2916          */
2917         gp = g_new_geomf(mp, "%s", md->md_name);
2918         sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
2919         gp->start = g_mirror_start;
2920         gp->orphan = g_mirror_orphan;
2921         gp->access = g_mirror_access;
2922         gp->dumpconf = g_mirror_dumpconf;
2923
2924         sc->sc_id = md->md_mid;
2925         sc->sc_slice = md->md_slice;
2926         sc->sc_balance = md->md_balance;
2927         sc->sc_mediasize = md->md_mediasize;
2928         sc->sc_sectorsize = md->md_sectorsize;
2929         sc->sc_ndisks = md->md_all;
2930         sc->sc_flags = md->md_mflags;
2931         sc->sc_bump_id = 0;
2932         sc->sc_idle = 1;
2933         sc->sc_last_write = time_uptime;
2934         sc->sc_writes = 0;
2935         sx_init(&sc->sc_lock, "gmirror:lock");
2936         bioq_init(&sc->sc_queue);
2937         mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
2938         bioq_init(&sc->sc_regular_delayed);
2939         bioq_init(&sc->sc_inflight);
2940         bioq_init(&sc->sc_sync_delayed);
2941         LIST_INIT(&sc->sc_disks);
2942         TAILQ_INIT(&sc->sc_events);
2943         mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
2944         callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
2945         mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
2946         sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
2947         gp->softc = sc;
2948         sc->sc_geom = gp;
2949         sc->sc_provider = NULL;
2950         /*
2951          * Synchronization geom.
2952          */
2953         gp = g_new_geomf(mp, "%s.sync", md->md_name);
2954         gp->softc = sc;
2955         gp->orphan = g_mirror_orphan;
2956         sc->sc_sync.ds_geom = gp;
2957         sc->sc_sync.ds_ndisks = 0;
2958         error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
2959             "g_mirror %s", md->md_name);
2960         if (error != 0) {
2961                 G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
2962                     sc->sc_name);
2963                 g_destroy_geom(sc->sc_sync.ds_geom);
2964                 mtx_destroy(&sc->sc_done_mtx);
2965                 mtx_destroy(&sc->sc_events_mtx);
2966                 mtx_destroy(&sc->sc_queue_mtx);
2967                 sx_destroy(&sc->sc_lock);
2968                 g_destroy_geom(sc->sc_geom);
2969                 free(sc, M_MIRROR);
2970                 return (NULL);
2971         }
2972
2973         G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
2974             sc->sc_name, sc->sc_ndisks, sc->sc_id);
2975
2976         sc->sc_rootmount = root_mount_hold("GMIRROR");
2977         G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
2978         /*
2979          * Run timeout.
2980          */
2981         timeout = g_mirror_timeout * hz;
2982         callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
2983         return (sc->sc_geom);
2984 }
2985
2986 int
2987 g_mirror_destroy(struct g_mirror_softc *sc, int how)
2988 {
2989         struct g_mirror_disk *disk;
2990         struct g_provider *pp;
2991
2992         g_topology_assert_not();
2993         if (sc == NULL)
2994                 return (ENXIO);
2995         sx_assert(&sc->sc_lock, SX_XLOCKED);
2996
2997         pp = sc->sc_provider;
2998         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
2999                 switch (how) {
3000                 case G_MIRROR_DESTROY_SOFT:
3001                         G_MIRROR_DEBUG(1,
3002                             "Device %s is still open (r%dw%de%d).", pp->name,
3003                             pp->acr, pp->acw, pp->ace);
3004                         return (EBUSY);
3005                 case G_MIRROR_DESTROY_DELAYED:
3006                         G_MIRROR_DEBUG(1,
3007                             "Device %s will be destroyed on last close.",
3008                             pp->name);
3009                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
3010                                 if (disk->d_state ==
3011                                     G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3012                                         g_mirror_sync_stop(disk, 1);
3013                                 }
3014                         }
3015                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
3016                         return (EBUSY);
3017                 case G_MIRROR_DESTROY_HARD:
3018                         G_MIRROR_DEBUG(1, "Device %s is still open, so it "
3019                             "can't be definitely removed.", pp->name);
3020                 }
3021         }
3022
3023         g_topology_lock();
3024         if (sc->sc_geom->softc == NULL) {
3025                 g_topology_unlock();
3026                 return (0);
3027         }
3028         sc->sc_geom->softc = NULL;
3029         sc->sc_sync.ds_geom->softc = NULL;
3030         g_topology_unlock();
3031
3032         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3033         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
3034         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3035         sx_xunlock(&sc->sc_lock);
3036         mtx_lock(&sc->sc_queue_mtx);
3037         wakeup(sc);
3038         mtx_unlock(&sc->sc_queue_mtx);
3039         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3040         while (sc->sc_worker != NULL)
3041                 tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3042         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3043         sx_xlock(&sc->sc_lock);
3044         g_mirror_destroy_device(sc);
3045         free(sc, M_MIRROR);
3046         return (0);
3047 }
3048
3049 static void
3050 g_mirror_taste_orphan(struct g_consumer *cp)
3051 {
3052
3053         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3054             cp->provider->name));
3055 }
3056
3057 static struct g_geom *
3058 g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3059 {
3060         struct g_mirror_metadata md;
3061         struct g_mirror_softc *sc;
3062         struct g_consumer *cp;
3063         struct g_geom *gp;
3064         int error;
3065
3066         g_topology_assert();
3067         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3068         G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3069
3070         gp = g_new_geomf(mp, "mirror:taste");
3071         /*
3072          * This orphan function should be never called.
3073          */
3074         gp->orphan = g_mirror_taste_orphan;
3075         cp = g_new_consumer(gp);
3076         g_attach(cp, pp);
3077         error = g_mirror_read_metadata(cp, &md);
3078         g_detach(cp);
3079         g_destroy_consumer(cp);
3080         g_destroy_geom(gp);
3081         if (error != 0)
3082                 return (NULL);
3083         gp = NULL;
3084
3085         if (md.md_provider[0] != '\0' &&
3086             !g_compare_names(md.md_provider, pp->name))
3087                 return (NULL);
3088         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3089                 return (NULL);
3090         if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3091                 G_MIRROR_DEBUG(0,
3092                     "Device %s: provider %s marked as inactive, skipping.",
3093                     md.md_name, pp->name);
3094                 return (NULL);
3095         }
3096         if (g_mirror_debug >= 2)
3097                 mirror_metadata_dump(&md);
3098
3099         /*
3100          * Let's check if device already exists.
3101          */
3102         sc = NULL;
3103         LIST_FOREACH(gp, &mp->geom, geom) {
3104                 sc = gp->softc;
3105                 if (sc == NULL)
3106                         continue;
3107                 if (sc->sc_sync.ds_geom == gp)
3108                         continue;
3109                 if (strcmp(md.md_name, sc->sc_name) != 0)
3110                         continue;
3111                 if (md.md_mid != sc->sc_id) {
3112                         G_MIRROR_DEBUG(0, "Device %s already configured.",
3113                             sc->sc_name);
3114                         return (NULL);
3115                 }
3116                 break;
3117         }
3118         if (gp == NULL) {
3119                 gp = g_mirror_create(mp, &md);
3120                 if (gp == NULL) {
3121                         G_MIRROR_DEBUG(0, "Cannot create device %s.",
3122                             md.md_name);
3123                         return (NULL);
3124                 }
3125                 sc = gp->softc;
3126         }
3127         G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3128         g_topology_unlock();
3129         sx_xlock(&sc->sc_lock);
3130         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3131         error = g_mirror_add_disk(sc, pp, &md);
3132         if (error != 0) {
3133                 G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3134                     pp->name, gp->name, error);
3135                 if (LIST_EMPTY(&sc->sc_disks)) {
3136                         g_cancel_event(sc);
3137                         g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3138                         g_topology_lock();
3139                         return (NULL);
3140                 }
3141                 gp = NULL;
3142         }
3143         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3144         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3145                 g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3146                 g_topology_lock();
3147                 return (NULL);
3148         }
3149         sx_xunlock(&sc->sc_lock);
3150         g_topology_lock();
3151         return (gp);
3152 }
3153
3154 static void
3155 g_mirror_resize(struct g_consumer *cp)
3156 {
3157         struct g_mirror_disk *disk;
3158
3159         g_topology_assert();
3160         g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3161
3162         disk = cp->private;
3163         if (disk == NULL)
3164                 return;
3165         g_topology_unlock();
3166         g_mirror_update_metadata(disk);
3167         g_topology_lock();
3168 }
3169
3170 static int
3171 g_mirror_destroy_geom(struct gctl_req *req __unused,
3172     struct g_class *mp __unused, struct g_geom *gp)
3173 {
3174         struct g_mirror_softc *sc;
3175         int error;
3176
3177         g_topology_unlock();
3178         sc = gp->softc;
3179         sx_xlock(&sc->sc_lock);
3180         g_cancel_event(sc);
3181         error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3182         if (error != 0)
3183                 sx_xunlock(&sc->sc_lock);
3184         g_topology_lock();
3185         return (error);
3186 }
3187
3188 static void
3189 g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3190     struct g_consumer *cp, struct g_provider *pp)
3191 {
3192         struct g_mirror_softc *sc;
3193
3194         g_topology_assert();
3195
3196         sc = gp->softc;
3197         if (sc == NULL)
3198                 return;
3199         /* Skip synchronization geom. */
3200         if (gp == sc->sc_sync.ds_geom)
3201                 return;
3202         if (pp != NULL) {
3203                 /* Nothing here. */
3204         } else if (cp != NULL) {
3205                 struct g_mirror_disk *disk;
3206
3207                 disk = cp->private;
3208                 if (disk == NULL)
3209                         return;
3210                 g_topology_unlock();
3211                 sx_xlock(&sc->sc_lock);
3212                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3213                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3214                         sbuf_printf(sb, "%s<Synchronized>", indent);
3215                         if (disk->d_sync.ds_offset == 0)
3216                                 sbuf_printf(sb, "0%%");
3217                         else {
3218                                 sbuf_printf(sb, "%u%%",
3219                                     (u_int)((disk->d_sync.ds_offset * 100) /
3220                                     sc->sc_provider->mediasize));
3221                         }
3222                         sbuf_printf(sb, "</Synchronized>\n");
3223                         if (disk->d_sync.ds_offset > 0) {
3224                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3225                                     "</BytesSynced>\n", indent,
3226                                     (intmax_t)disk->d_sync.ds_offset);
3227                         }
3228                 }
3229                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3230                     disk->d_sync.ds_syncid);
3231                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3232                     disk->d_genid);
3233                 sbuf_printf(sb, "%s<Flags>", indent);
3234                 if (disk->d_flags == 0)
3235                         sbuf_printf(sb, "NONE");
3236                 else {
3237                         int first = 1;
3238
3239 #define ADD_FLAG(flag, name)    do {                                    \
3240         if ((disk->d_flags & (flag)) != 0) {                            \
3241                 if (!first)                                             \
3242                         sbuf_printf(sb, ", ");                          \
3243                 else                                                    \
3244                         first = 0;                                      \
3245                 sbuf_printf(sb, name);                                  \
3246         }                                                               \
3247 } while (0)
3248                         ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3249                         ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3250                         ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3251                         ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3252                             "SYNCHRONIZING");
3253                         ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3254                         ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3255 #undef  ADD_FLAG
3256                 }
3257                 sbuf_printf(sb, "</Flags>\n");
3258                 sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3259                     disk->d_priority);
3260                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3261                     g_mirror_disk_state2str(disk->d_state));
3262                 sx_xunlock(&sc->sc_lock);
3263                 g_topology_lock();
3264         } else {
3265                 g_topology_unlock();
3266                 sx_xlock(&sc->sc_lock);
3267                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3268                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3269                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3270                 sbuf_printf(sb, "%s<Flags>", indent);
3271                 if (sc->sc_flags == 0)
3272                         sbuf_printf(sb, "NONE");
3273                 else {
3274                         int first = 1;
3275
3276 #define ADD_FLAG(flag, name)    do {                                    \
3277         if ((sc->sc_flags & (flag)) != 0) {                             \
3278                 if (!first)                                             \
3279                         sbuf_printf(sb, ", ");                          \
3280                 else                                                    \
3281                         first = 0;                                      \
3282                 sbuf_printf(sb, name);                                  \
3283         }                                                               \
3284 } while (0)
3285                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3286                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3287 #undef  ADD_FLAG
3288                 }
3289                 sbuf_printf(sb, "</Flags>\n");
3290                 sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3291                     (u_int)sc->sc_slice);
3292                 sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3293                     balance_name(sc->sc_balance));
3294                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3295                     sc->sc_ndisks);
3296                 sbuf_printf(sb, "%s<State>", indent);
3297                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3298                         sbuf_printf(sb, "%s", "STARTING");
3299                 else if (sc->sc_ndisks ==
3300                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3301                         sbuf_printf(sb, "%s", "COMPLETE");
3302                 else
3303                         sbuf_printf(sb, "%s", "DEGRADED");
3304                 sbuf_printf(sb, "</State>\n");
3305                 sx_xunlock(&sc->sc_lock);
3306                 g_topology_lock();
3307         }
3308 }
3309
3310 static void
3311 g_mirror_shutdown_post_sync(void *arg, int howto)
3312 {
3313         struct g_class *mp;
3314         struct g_geom *gp, *gp2;
3315         struct g_mirror_softc *sc;
3316         int error;
3317
3318         mp = arg;
3319         DROP_GIANT();
3320         g_topology_lock();
3321         g_mirror_shutdown = 1;
3322         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3323                 if ((sc = gp->softc) == NULL)
3324                         continue;
3325                 /* Skip synchronization geom. */
3326                 if (gp == sc->sc_sync.ds_geom)
3327                         continue;
3328                 g_topology_unlock();
3329                 sx_xlock(&sc->sc_lock);
3330                 g_mirror_idle(sc, -1);
3331                 g_cancel_event(sc);
3332                 error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3333                 if (error != 0)
3334                         sx_xunlock(&sc->sc_lock);
3335                 g_topology_lock();
3336         }
3337         g_topology_unlock();
3338         PICKUP_GIANT();
3339 }
3340
3341 static void
3342 g_mirror_init(struct g_class *mp)
3343 {
3344
3345         g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3346             g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3347         if (g_mirror_post_sync == NULL)
3348                 G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3349 }
3350
3351 static void
3352 g_mirror_fini(struct g_class *mp)
3353 {
3354
3355         if (g_mirror_post_sync != NULL)
3356                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3357 }
3358
3359 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);