sys/geom/mirror/g_mirror.c

   1 /*-
   2  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/module.h>
  34 #include <sys/limits.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/bio.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/malloc.h>
  41 #include <sys/eventhandler.h>
  42 #include <vm/uma.h>
  43 #include <geom/geom.h>
  44 #include <sys/proc.h>
  45 #include <sys/kthread.h>
  46 #include <sys/sched.h>
  47 #include <geom/mirror/g_mirror.h>
  48
  49 FEATURE(geom_mirror, "GEOM mirroring support");
  50
  51 static MALLOC_DEFINE(M_MIRROR, "mirror_data", "GEOM_MIRROR Data");
  52
  53 SYSCTL_DECL(_kern_geom);
  54 static SYSCTL_NODE(_kern_geom, OID_AUTO, mirror, CTLFLAG_RW, 0,
  55     "GEOM_MIRROR stuff");
  56 u_int g_mirror_debug = 0;
  57 TUNABLE_INT("kern.geom.mirror.debug", &g_mirror_debug);
  58 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, debug, CTLFLAG_RW, &g_mirror_debug, 0,
  59     "Debug level");
  60 static u_int g_mirror_timeout = 4;
  61 TUNABLE_INT("kern.geom.mirror.timeout", &g_mirror_timeout);
  62 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, timeout, CTLFLAG_RW, &g_mirror_timeout,
  63     0, "Time to wait on all mirror components");
  64 static u_int g_mirror_idletime = 5;
  65 TUNABLE_INT("kern.geom.mirror.idletime", &g_mirror_idletime);
  66 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, idletime, CTLFLAG_RW,
  67     &g_mirror_idletime, 0, "Mark components as clean when idling");
  68 static u_int g_mirror_disconnect_on_failure = 1;
  69 TUNABLE_INT("kern.geom.mirror.disconnect_on_failure",
  70     &g_mirror_disconnect_on_failure);
  71 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, disconnect_on_failure, CTLFLAG_RW,
  72     &g_mirror_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  73 static u_int g_mirror_syncreqs = 2;
  74 TUNABLE_INT("kern.geom.mirror.sync_requests", &g_mirror_syncreqs);
  75 SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  76     &g_mirror_syncreqs, 0, "Parallel synchronization I/O requests.");
  77
  78 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  79         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));       \
  80         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  81         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));       \
  82 } while (0)
  83
  84 static eventhandler_tag g_mirror_post_sync = NULL;
  85 static int g_mirror_shutdown = 0;
  86
  87 static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
  88     struct g_geom *gp);
  89 static g_taste_t g_mirror_taste;
  90 static g_resize_t g_mirror_resize;
  91 static void g_mirror_init(struct g_class *mp);
  92 static void g_mirror_fini(struct g_class *mp);
  93
  94 struct g_class g_mirror_class = {
  95         .name = G_MIRROR_CLASS_NAME,
  96         .version = G_VERSION,
  97         .ctlreq = g_mirror_config,
  98         .taste = g_mirror_taste,
  99         .destroy_geom = g_mirror_destroy_geom,
 100         .init = g_mirror_init,
 101         .fini = g_mirror_fini,
 102         .resize = g_mirror_resize
 103 };
 104
 105
 106 static void g_mirror_destroy_provider(struct g_mirror_softc *sc);
 107 static int g_mirror_update_disk(struct g_mirror_disk *disk, u_int state);
 108 static void g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force);
 109 static void g_mirror_dumpconf(struct sbuf *sb, const char *indent,
 110     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 111 static void g_mirror_sync_stop(struct g_mirror_disk *disk, int type);
 112 static void g_mirror_register_request(struct bio *bp);
 113 static void g_mirror_sync_release(struct g_mirror_softc *sc);
 114
 115
 116 static const char *
 117 g_mirror_disk_state2str(int state)
 118 {
 119
 120         switch (state) {
 121         case G_MIRROR_DISK_STATE_NONE:
 122                 return ("NONE");
 123         case G_MIRROR_DISK_STATE_NEW:
 124                 return ("NEW");
 125         case G_MIRROR_DISK_STATE_ACTIVE:
 126                 return ("ACTIVE");
 127         case G_MIRROR_DISK_STATE_STALE:
 128                 return ("STALE");
 129         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 130                 return ("SYNCHRONIZING");
 131         case G_MIRROR_DISK_STATE_DISCONNECTED:
 132                 return ("DISCONNECTED");
 133         case G_MIRROR_DISK_STATE_DESTROY:
 134                 return ("DESTROY");
 135         default:
 136                 return ("INVALID");
 137         }
 138 }
 139
 140 static const char *
 141 g_mirror_device_state2str(int state)
 142 {
 143
 144         switch (state) {
 145         case G_MIRROR_DEVICE_STATE_STARTING:
 146                 return ("STARTING");
 147         case G_MIRROR_DEVICE_STATE_RUNNING:
 148                 return ("RUNNING");
 149         default:
 150                 return ("INVALID");
 151         }
 152 }
 153
 154 static const char *
 155 g_mirror_get_diskname(struct g_mirror_disk *disk)
 156 {
 157
 158         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 159                 return ("[unknown]");
 160         return (disk->d_name);
 161 }
 162
 163 /*
 164  * --- Events handling functions ---
 165  * Events in geom_mirror are used to maintain disks and device status
 166  * from one thread to simplify locking.
 167  */
 168 static void
 169 g_mirror_event_free(struct g_mirror_event *ep)
 170 {
 171
 172         free(ep, M_MIRROR);
 173 }
 174
 175 int
 176 g_mirror_event_send(void *arg, int state, int flags)
 177 {
 178         struct g_mirror_softc *sc;
 179         struct g_mirror_disk *disk;
 180         struct g_mirror_event *ep;
 181         int error;
 182
 183         ep = malloc(sizeof(*ep), M_MIRROR, M_WAITOK);
 184         G_MIRROR_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 185         if ((flags & G_MIRROR_EVENT_DEVICE) != 0) {
 186                 disk = NULL;
 187                 sc = arg;
 188         } else {
 189                 disk = arg;
 190                 sc = disk->d_softc;
 191         }
 192         ep->e_disk = disk;
 193         ep->e_state = state;
 194         ep->e_flags = flags;
 195         ep->e_error = 0;
 196         mtx_lock(&sc->sc_events_mtx);
 197         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 198         mtx_unlock(&sc->sc_events_mtx);
 199         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 200         mtx_lock(&sc->sc_queue_mtx);
 201         wakeup(sc);
 202         mtx_unlock(&sc->sc_queue_mtx);
 203         if ((flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 204                 return (0);
 205         sx_assert(&sc->sc_lock, SX_XLOCKED);
 206         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 207         sx_xunlock(&sc->sc_lock);
 208         while ((ep->e_flags & G_MIRROR_EVENT_DONE) == 0) {
 209                 mtx_lock(&sc->sc_events_mtx);
 210                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "m:event",
 211                     hz * 5);
 212         }
 213         error = ep->e_error;
 214         g_mirror_event_free(ep);
 215         sx_xlock(&sc->sc_lock);
 216         return (error);
 217 }
 218
 219 static struct g_mirror_event *
 220 g_mirror_event_get(struct g_mirror_softc *sc)
 221 {
 222         struct g_mirror_event *ep;
 223
 224         mtx_lock(&sc->sc_events_mtx);
 225         ep = TAILQ_FIRST(&sc->sc_events);
 226         mtx_unlock(&sc->sc_events_mtx);
 227         return (ep);
 228 }
 229
 230 static void
 231 g_mirror_event_remove(struct g_mirror_softc *sc, struct g_mirror_event *ep)
 232 {
 233
 234         mtx_lock(&sc->sc_events_mtx);
 235         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 236         mtx_unlock(&sc->sc_events_mtx);
 237 }
 238
 239 static void
 240 g_mirror_event_cancel(struct g_mirror_disk *disk)
 241 {
 242         struct g_mirror_softc *sc;
 243         struct g_mirror_event *ep, *tmpep;
 244
 245         sc = disk->d_softc;
 246         sx_assert(&sc->sc_lock, SX_XLOCKED);
 247
 248         mtx_lock(&sc->sc_events_mtx);
 249         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 250                 if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0)
 251                         continue;
 252                 if (ep->e_disk != disk)
 253                         continue;
 254                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 255                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 256                         g_mirror_event_free(ep);
 257                 else {
 258                         ep->e_error = ECANCELED;
 259                         wakeup(ep);
 260                 }
 261         }
 262         mtx_unlock(&sc->sc_events_mtx);
 263 }
 264
 265 /*
 266  * Return the number of disks in given state.
 267  * If state is equal to -1, count all connected disks.
 268  */
 269 u_int
 270 g_mirror_ndisks(struct g_mirror_softc *sc, int state)
 271 {
 272         struct g_mirror_disk *disk;
 273         u_int n = 0;
 274
 275         sx_assert(&sc->sc_lock, SX_LOCKED);
 276
 277         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 278                 if (state == -1 || disk->d_state == state)
 279                         n++;
 280         }
 281         return (n);
 282 }
 283
 284 /*
 285  * Find a disk in mirror by its disk ID.
 286  */
 287 static struct g_mirror_disk *
 288 g_mirror_id2disk(struct g_mirror_softc *sc, uint32_t id)
 289 {
 290         struct g_mirror_disk *disk;
 291
 292         sx_assert(&sc->sc_lock, SX_XLOCKED);
 293
 294         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 295                 if (disk->d_id == id)
 296                         return (disk);
 297         }
 298         return (NULL);
 299 }
 300
 301 static u_int
 302 g_mirror_nrequests(struct g_mirror_softc *sc, struct g_consumer *cp)
 303 {
 304         struct bio *bp;
 305         u_int nreqs = 0;
 306
 307         mtx_lock(&sc->sc_queue_mtx);
 308         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 309                 if (bp->bio_from == cp)
 310                         nreqs++;
 311         }
 312         mtx_unlock(&sc->sc_queue_mtx);
 313         return (nreqs);
 314 }
 315
 316 static int
 317 g_mirror_is_busy(struct g_mirror_softc *sc, struct g_consumer *cp)
 318 {
 319
 320         if (cp->index > 0) {
 321                 G_MIRROR_DEBUG(2,
 322                     "I/O requests for %s exist, can't destroy it now.",
 323                     cp->provider->name);
 324                 return (1);
 325         }
 326         if (g_mirror_nrequests(sc, cp) > 0) {
 327                 G_MIRROR_DEBUG(2,
 328                     "I/O requests for %s in queue, can't destroy it now.",
 329                     cp->provider->name);
 330                 return (1);
 331         }
 332         return (0);
 333 }
 334
 335 static void
 336 g_mirror_destroy_consumer(void *arg, int flags __unused)
 337 {
 338         struct g_consumer *cp;
 339
 340         g_topology_assert();
 341
 342         cp = arg;
 343         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 344         g_detach(cp);
 345         g_destroy_consumer(cp);
 346 }
 347
 348 static void
 349 g_mirror_kill_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 350 {
 351         struct g_provider *pp;
 352         int retaste_wait;
 353
 354         g_topology_assert();
 355
 356         cp->private = NULL;
 357         if (g_mirror_is_busy(sc, cp))
 358                 return;
 359         pp = cp->provider;
 360         retaste_wait = 0;
 361         if (cp->acw == 1) {
 362                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 363                         retaste_wait = 1;
 364         }
 365         G_MIRROR_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 366             -cp->acw, -cp->ace, 0);
 367         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 368                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 369         if (retaste_wait) {
 370                 /*
 371                  * After retaste event was send (inside g_access()), we can send
 372                  * event to detach and destroy consumer.
 373                  * A class, which has consumer to the given provider connected
 374                  * will not receive retaste event for the provider.
 375                  * This is the way how I ignore retaste events when I close
 376                  * consumers opened for write: I detach and destroy consumer
 377                  * after retaste event is sent.
 378                  */
 379                 g_post_event(g_mirror_destroy_consumer, cp, M_WAITOK, NULL);
 380                 return;
 381         }
 382         G_MIRROR_DEBUG(1, "Consumer %s destroyed.", pp->name);
 383         g_detach(cp);
 384         g_destroy_consumer(cp);
 385 }
 386
 387 static int
 388 g_mirror_connect_disk(struct g_mirror_disk *disk, struct g_provider *pp)
 389 {
 390         struct g_consumer *cp;
 391         int error;
 392
 393         g_topology_assert_not();
 394         KASSERT(disk->d_consumer == NULL,
 395             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 396
 397         g_topology_lock();
 398         cp = g_new_consumer(disk->d_softc->sc_geom);
 399         cp->flags |= G_CF_DIRECT_RECEIVE;
 400         error = g_attach(cp, pp);
 401         if (error != 0) {
 402                 g_destroy_consumer(cp);
 403                 g_topology_unlock();
 404                 return (error);
 405         }
 406         error = g_access(cp, 1, 1, 1);
 407         if (error != 0) {
 408                 g_detach(cp);
 409                 g_destroy_consumer(cp);
 410                 g_topology_unlock();
 411                 G_MIRROR_DEBUG(0, "Cannot open consumer %s (error=%d).",
 412                     pp->name, error);
 413                 return (error);
 414         }
 415         g_topology_unlock();
 416         disk->d_consumer = cp;
 417         disk->d_consumer->private = disk;
 418         disk->d_consumer->index = 0;
 419
 420         G_MIRROR_DEBUG(2, "Disk %s connected.", g_mirror_get_diskname(disk));
 421         return (0);
 422 }
 423
 424 static void
 425 g_mirror_disconnect_consumer(struct g_mirror_softc *sc, struct g_consumer *cp)
 426 {
 427
 428         g_topology_assert();
 429
 430         if (cp == NULL)
 431                 return;
 432         if (cp->provider != NULL)
 433                 g_mirror_kill_consumer(sc, cp);
 434         else
 435                 g_destroy_consumer(cp);
 436 }
 437
 438 /*
 439  * Initialize disk. This means allocate memory, create consumer, attach it
 440  * to the provider and open access (r1w1e1) to it.
 441  */
 442 static struct g_mirror_disk *
 443 g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 444     struct g_mirror_metadata *md, int *errorp)
 445 {
 446         struct g_mirror_disk *disk;
 447         int i, error;
 448
 449         disk = malloc(sizeof(*disk), M_MIRROR, M_NOWAIT | M_ZERO);
 450         if (disk == NULL) {
 451                 error = ENOMEM;
 452                 goto fail;
 453         }
 454         disk->d_softc = sc;
 455         error = g_mirror_connect_disk(disk, pp);
 456         if (error != 0)
 457                 goto fail;
 458         disk->d_id = md->md_did;
 459         disk->d_state = G_MIRROR_DISK_STATE_NONE;
 460         disk->d_priority = md->md_priority;
 461         disk->d_flags = md->md_dflags;
 462         error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
 463         if (error == 0 && i != 0)
 464                 disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 465         if (md->md_provider[0] != '\0')
 466                 disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
 467         disk->d_sync.ds_consumer = NULL;
 468         disk->d_sync.ds_offset = md->md_sync_offset;
 469         disk->d_sync.ds_offset_done = md->md_sync_offset;
 470         disk->d_genid = md->md_genid;
 471         disk->d_sync.ds_syncid = md->md_syncid;
 472         if (errorp != NULL)
 473                 *errorp = 0;
 474         return (disk);
 475 fail:
 476         if (errorp != NULL)
 477                 *errorp = error;
 478         if (disk != NULL)
 479                 free(disk, M_MIRROR);
 480         return (NULL);
 481 }
 482
 483 static void
 484 g_mirror_destroy_disk(struct g_mirror_disk *disk)
 485 {
 486         struct g_mirror_softc *sc;
 487
 488         g_topology_assert_not();
 489         sc = disk->d_softc;
 490         sx_assert(&sc->sc_lock, SX_XLOCKED);
 491
 492         LIST_REMOVE(disk, d_next);
 493         g_mirror_event_cancel(disk);
 494         if (sc->sc_hint == disk)
 495                 sc->sc_hint = NULL;
 496         switch (disk->d_state) {
 497         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
 498                 g_mirror_sync_stop(disk, 1);
 499                 /* FALLTHROUGH */
 500         case G_MIRROR_DISK_STATE_NEW:
 501         case G_MIRROR_DISK_STATE_STALE:
 502         case G_MIRROR_DISK_STATE_ACTIVE:
 503                 g_topology_lock();
 504                 g_mirror_disconnect_consumer(sc, disk->d_consumer);
 505                 g_topology_unlock();
 506                 free(disk, M_MIRROR);
 507                 break;
 508         default:
 509                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 510                     g_mirror_get_diskname(disk),
 511                     g_mirror_disk_state2str(disk->d_state)));
 512         }
 513 }
 514
 515 static void
 516 g_mirror_destroy_device(struct g_mirror_softc *sc)
 517 {
 518         struct g_mirror_disk *disk;
 519         struct g_mirror_event *ep;
 520         struct g_geom *gp;
 521         struct g_consumer *cp, *tmpcp;
 522
 523         g_topology_assert_not();
 524         sx_assert(&sc->sc_lock, SX_XLOCKED);
 525
 526         gp = sc->sc_geom;
 527         if (sc->sc_provider != NULL)
 528                 g_mirror_destroy_provider(sc);
 529         for (disk = LIST_FIRST(&sc->sc_disks); disk != NULL;
 530             disk = LIST_FIRST(&sc->sc_disks)) {
 531                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 532                 g_mirror_update_metadata(disk);
 533                 g_mirror_destroy_disk(disk);
 534         }
 535         while ((ep = g_mirror_event_get(sc)) != NULL) {
 536                 g_mirror_event_remove(sc, ep);
 537                 if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0)
 538                         g_mirror_event_free(ep);
 539                 else {
 540                         ep->e_error = ECANCELED;
 541                         ep->e_flags |= G_MIRROR_EVENT_DONE;
 542                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 543                         mtx_lock(&sc->sc_events_mtx);
 544                         wakeup(ep);
 545                         mtx_unlock(&sc->sc_events_mtx);
 546                 }
 547         }
 548         callout_drain(&sc->sc_callout);
 549
 550         g_topology_lock();
 551         LIST_FOREACH_SAFE(cp, &sc->sc_sync.ds_geom->consumer, consumer, tmpcp) {
 552                 g_mirror_disconnect_consumer(sc, cp);
 553         }
 554         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 555         G_MIRROR_DEBUG(0, "Device %s destroyed.", gp->name);
 556         g_wither_geom(gp, ENXIO);
 557         g_topology_unlock();
 558         mtx_destroy(&sc->sc_queue_mtx);
 559         mtx_destroy(&sc->sc_events_mtx);
 560         mtx_destroy(&sc->sc_done_mtx);
 561         sx_xunlock(&sc->sc_lock);
 562         sx_destroy(&sc->sc_lock);
 563 }
 564
 565 static void
 566 g_mirror_orphan(struct g_consumer *cp)
 567 {
 568         struct g_mirror_disk *disk;
 569
 570         g_topology_assert();
 571
 572         disk = cp->private;
 573         if (disk == NULL)
 574                 return;
 575         disk->d_softc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
 576         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_DISCONNECTED,
 577             G_MIRROR_EVENT_DONTWAIT);
 578 }
 579
 580 /*
 581  * Function should return the next active disk on the list.
 582  * It is possible that it will be the same disk as given.
 583  * If there are no active disks on list, NULL is returned.
 584  */
 585 static __inline struct g_mirror_disk *
 586 g_mirror_find_next(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
 587 {
 588         struct g_mirror_disk *dp;
 589
 590         for (dp = LIST_NEXT(disk, d_next); dp != disk;
 591             dp = LIST_NEXT(dp, d_next)) {
 592                 if (dp == NULL)
 593                         dp = LIST_FIRST(&sc->sc_disks);
 594                 if (dp->d_state == G_MIRROR_DISK_STATE_ACTIVE)
 595                         break;
 596         }
 597         if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 598                 return (NULL);
 599         return (dp);
 600 }
 601
 602 static struct g_mirror_disk *
 603 g_mirror_get_disk(struct g_mirror_softc *sc)
 604 {
 605         struct g_mirror_disk *disk;
 606
 607         if (sc->sc_hint == NULL) {
 608                 sc->sc_hint = LIST_FIRST(&sc->sc_disks);
 609                 if (sc->sc_hint == NULL)
 610                         return (NULL);
 611         }
 612         disk = sc->sc_hint;
 613         if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE) {
 614                 disk = g_mirror_find_next(sc, disk);
 615                 if (disk == NULL)
 616                         return (NULL);
 617         }
 618         sc->sc_hint = g_mirror_find_next(sc, disk);
 619         return (disk);
 620 }
 621
 622 static int
 623 g_mirror_write_metadata(struct g_mirror_disk *disk,
 624     struct g_mirror_metadata *md)
 625 {
 626         struct g_mirror_softc *sc;
 627         struct g_consumer *cp;
 628         off_t offset, length;
 629         u_char *sector;
 630         int error = 0;
 631
 632         g_topology_assert_not();
 633         sc = disk->d_softc;
 634         sx_assert(&sc->sc_lock, SX_LOCKED);
 635
 636         cp = disk->d_consumer;
 637         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 638         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 639         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 640             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 641             cp->acw, cp->ace));
 642         length = cp->provider->sectorsize;
 643         offset = cp->provider->mediasize - length;
 644         sector = malloc((size_t)length, M_MIRROR, M_WAITOK | M_ZERO);
 645         if (md != NULL &&
 646             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0) {
 647                 /*
 648                  * Handle the case, when the size of parent provider reduced.
 649                  */
 650                 if (offset < md->md_mediasize)
 651                         error = ENOSPC;
 652                 else
 653                         mirror_metadata_encode(md, sector);
 654         }
 655         if (error == 0)
 656                 error = g_write_data(cp, offset, sector, length);
 657         free(sector, M_MIRROR);
 658         if (error != 0) {
 659                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 660                         disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 661                         G_MIRROR_DEBUG(0, "Cannot write metadata on %s "
 662                             "(device=%s, error=%d).",
 663                             g_mirror_get_diskname(disk), sc->sc_name, error);
 664                 } else {
 665                         G_MIRROR_DEBUG(1, "Cannot write metadata on %s "
 666                             "(device=%s, error=%d).",
 667                             g_mirror_get_diskname(disk), sc->sc_name, error);
 668                 }
 669                 if (g_mirror_disconnect_on_failure &&
 670                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1) {
 671                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 672                         g_mirror_event_send(disk,
 673                             G_MIRROR_DISK_STATE_DISCONNECTED,
 674                             G_MIRROR_EVENT_DONTWAIT);
 675                 }
 676         }
 677         return (error);
 678 }
 679
 680 static int
 681 g_mirror_clear_metadata(struct g_mirror_disk *disk)
 682 {
 683         int error;
 684
 685         g_topology_assert_not();
 686         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 687
 688         error = g_mirror_write_metadata(disk, NULL);
 689         if (error == 0) {
 690                 G_MIRROR_DEBUG(2, "Metadata on %s cleared.",
 691                     g_mirror_get_diskname(disk));
 692         } else {
 693                 G_MIRROR_DEBUG(0,
 694                     "Cannot clear metadata on disk %s (error=%d).",
 695                     g_mirror_get_diskname(disk), error);
 696         }
 697         return (error);
 698 }
 699
 700 void
 701 g_mirror_fill_metadata(struct g_mirror_softc *sc, struct g_mirror_disk *disk,
 702     struct g_mirror_metadata *md)
 703 {
 704
 705         strlcpy(md->md_magic, G_MIRROR_MAGIC, sizeof(md->md_magic));
 706         md->md_version = G_MIRROR_VERSION;
 707         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 708         md->md_mid = sc->sc_id;
 709         md->md_all = sc->sc_ndisks;
 710         md->md_slice = sc->sc_slice;
 711         md->md_balance = sc->sc_balance;
 712         md->md_genid = sc->sc_genid;
 713         md->md_mediasize = sc->sc_mediasize;
 714         md->md_sectorsize = sc->sc_sectorsize;
 715         md->md_mflags = (sc->sc_flags & G_MIRROR_DEVICE_FLAG_MASK);
 716         bzero(md->md_provider, sizeof(md->md_provider));
 717         if (disk == NULL) {
 718                 md->md_did = arc4random();
 719                 md->md_priority = 0;
 720                 md->md_syncid = 0;
 721                 md->md_dflags = 0;
 722                 md->md_sync_offset = 0;
 723                 md->md_provsize = 0;
 724         } else {
 725                 md->md_did = disk->d_id;
 726                 md->md_priority = disk->d_priority;
 727                 md->md_syncid = disk->d_sync.ds_syncid;
 728                 md->md_dflags = (disk->d_flags & G_MIRROR_DISK_FLAG_MASK);
 729                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
 730                         md->md_sync_offset = disk->d_sync.ds_offset_done;
 731                 else
 732                         md->md_sync_offset = 0;
 733                 if ((disk->d_flags & G_MIRROR_DISK_FLAG_HARDCODED) != 0) {
 734                         strlcpy(md->md_provider,
 735                             disk->d_consumer->provider->name,
 736                             sizeof(md->md_provider));
 737                 }
 738                 md->md_provsize = disk->d_consumer->provider->mediasize;
 739         }
 740 }
 741
 742 void
 743 g_mirror_update_metadata(struct g_mirror_disk *disk)
 744 {
 745         struct g_mirror_softc *sc;
 746         struct g_mirror_metadata md;
 747         int error;
 748
 749         g_topology_assert_not();
 750         sc = disk->d_softc;
 751         sx_assert(&sc->sc_lock, SX_LOCKED);
 752
 753         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WIPE) == 0)
 754                 g_mirror_fill_metadata(sc, disk, &md);
 755         error = g_mirror_write_metadata(disk, &md);
 756         if (error == 0) {
 757                 G_MIRROR_DEBUG(2, "Metadata on %s updated.",
 758                     g_mirror_get_diskname(disk));
 759         } else {
 760                 G_MIRROR_DEBUG(0,
 761                     "Cannot update metadata on disk %s (error=%d).",
 762                     g_mirror_get_diskname(disk), error);
 763         }
 764 }
 765
 766 static void
 767 g_mirror_bump_syncid(struct g_mirror_softc *sc)
 768 {
 769         struct g_mirror_disk *disk;
 770
 771         g_topology_assert_not();
 772         sx_assert(&sc->sc_lock, SX_XLOCKED);
 773         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 774             ("%s called with no active disks (device=%s).", __func__,
 775             sc->sc_name));
 776
 777         sc->sc_syncid++;
 778         G_MIRROR_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 779             sc->sc_syncid);
 780         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 781                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 782                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 783                         disk->d_sync.ds_syncid = sc->sc_syncid;
 784                         g_mirror_update_metadata(disk);
 785                 }
 786         }
 787 }
 788
 789 static void
 790 g_mirror_bump_genid(struct g_mirror_softc *sc)
 791 {
 792         struct g_mirror_disk *disk;
 793
 794         g_topology_assert_not();
 795         sx_assert(&sc->sc_lock, SX_XLOCKED);
 796         KASSERT(g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 0,
 797             ("%s called with no active disks (device=%s).", __func__,
 798             sc->sc_name));
 799
 800         sc->sc_genid++;
 801         G_MIRROR_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 802             sc->sc_genid);
 803         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 804                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
 805                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
 806                         disk->d_genid = sc->sc_genid;
 807                         g_mirror_update_metadata(disk);
 808                 }
 809         }
 810 }
 811
 812 static int
 813 g_mirror_idle(struct g_mirror_softc *sc, int acw)
 814 {
 815         struct g_mirror_disk *disk;
 816         int timeout;
 817
 818         g_topology_assert_not();
 819         sx_assert(&sc->sc_lock, SX_XLOCKED);
 820
 821         if (sc->sc_provider == NULL)
 822                 return (0);
 823         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 824                 return (0);
 825         if (sc->sc_idle)
 826                 return (0);
 827         if (sc->sc_writes > 0)
 828                 return (0);
 829         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 830                 timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
 831                 if (!g_mirror_shutdown && timeout > 0)
 832                         return (timeout);
 833         }
 834         sc->sc_idle = 1;
 835         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 836                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 837                         continue;
 838                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
 839                     g_mirror_get_diskname(disk), sc->sc_name);
 840                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
 841                 g_mirror_update_metadata(disk);
 842         }
 843         return (0);
 844 }
 845
 846 static void
 847 g_mirror_unidle(struct g_mirror_softc *sc)
 848 {
 849         struct g_mirror_disk *disk;
 850
 851         g_topology_assert_not();
 852         sx_assert(&sc->sc_lock, SX_XLOCKED);
 853
 854         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
 855                 return;
 856         sc->sc_idle = 0;
 857         sc->sc_last_write = time_uptime;
 858         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
 859                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
 860                         continue;
 861                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
 862                     g_mirror_get_diskname(disk), sc->sc_name);
 863                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
 864                 g_mirror_update_metadata(disk);
 865         }
 866 }
 867
 868 static void
 869 g_mirror_flush_done(struct bio *bp)
 870 {
 871         struct g_mirror_softc *sc;
 872         struct bio *pbp;
 873
 874         pbp = bp->bio_parent;
 875         sc = pbp->bio_to->geom->softc;
 876         mtx_lock(&sc->sc_done_mtx);
 877         if (pbp->bio_error == 0)
 878                 pbp->bio_error = bp->bio_error;
 879         pbp->bio_completed += bp->bio_completed;
 880         pbp->bio_inbed++;
 881         if (pbp->bio_children == pbp->bio_inbed) {
 882                 mtx_unlock(&sc->sc_done_mtx);
 883                 g_io_deliver(pbp, pbp->bio_error);
 884         } else
 885                 mtx_unlock(&sc->sc_done_mtx);
 886         g_destroy_bio(bp);
 887 }
 888
 889 static void
 890 g_mirror_done(struct bio *bp)
 891 {
 892         struct g_mirror_softc *sc;
 893
 894         sc = bp->bio_from->geom->softc;
 895         bp->bio_cflags = G_MIRROR_BIO_FLAG_REGULAR;
 896         mtx_lock(&sc->sc_queue_mtx);
 897         bioq_disksort(&sc->sc_queue, bp);
 898         mtx_unlock(&sc->sc_queue_mtx);
 899         wakeup(sc);
 900 }
 901
 902 static void
 903 g_mirror_regular_request(struct bio *bp)
 904 {
 905         struct g_mirror_softc *sc;
 906         struct g_mirror_disk *disk;
 907         struct bio *pbp;
 908
 909         g_topology_assert_not();
 910
 911         pbp = bp->bio_parent;
 912         sc = pbp->bio_to->geom->softc;
 913         bp->bio_from->index--;
 914         if (bp->bio_cmd == BIO_WRITE)
 915                 sc->sc_writes--;
 916         disk = bp->bio_from->private;
 917         if (disk == NULL) {
 918                 g_topology_lock();
 919                 g_mirror_kill_consumer(sc, bp->bio_from);
 920                 g_topology_unlock();
 921         }
 922
 923         pbp->bio_inbed++;
 924         KASSERT(pbp->bio_inbed <= pbp->bio_children,
 925             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
 926             pbp->bio_children));
 927         if (bp->bio_error == 0 && pbp->bio_error == 0) {
 928                 G_MIRROR_LOGREQ(3, bp, "Request delivered.");
 929                 g_destroy_bio(bp);
 930                 if (pbp->bio_children == pbp->bio_inbed) {
 931                         G_MIRROR_LOGREQ(3, pbp, "Request delivered.");
 932                         pbp->bio_completed = pbp->bio_length;
 933                         if (pbp->bio_cmd == BIO_WRITE ||
 934                             pbp->bio_cmd == BIO_DELETE) {
 935                                 bioq_remove(&sc->sc_inflight, pbp);
 936                                 /* Release delayed sync requests if possible. */
 937                                 g_mirror_sync_release(sc);
 938                         }
 939                         g_io_deliver(pbp, pbp->bio_error);
 940                 }
 941                 return;
 942         } else if (bp->bio_error != 0) {
 943                 if (pbp->bio_error == 0)
 944                         pbp->bio_error = bp->bio_error;
 945                 if (disk != NULL) {
 946                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_BROKEN) == 0) {
 947                                 disk->d_flags |= G_MIRROR_DISK_FLAG_BROKEN;
 948                                 G_MIRROR_LOGREQ(0, bp,
 949                                     "Request failed (error=%d).",
 950                                     bp->bio_error);
 951                         } else {
 952                                 G_MIRROR_LOGREQ(1, bp,
 953                                     "Request failed (error=%d).",
 954                                     bp->bio_error);
 955                         }
 956                         if (g_mirror_disconnect_on_failure &&
 957                             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) > 1)
 958                         {
 959                                 sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
 960                                 g_mirror_event_send(disk,
 961                                     G_MIRROR_DISK_STATE_DISCONNECTED,
 962                                     G_MIRROR_EVENT_DONTWAIT);
 963                         }
 964                 }
 965                 switch (pbp->bio_cmd) {
 966                 case BIO_DELETE:
 967                 case BIO_WRITE:
 968                         pbp->bio_inbed--;
 969                         pbp->bio_children--;
 970                         break;
 971                 }
 972         }
 973         g_destroy_bio(bp);
 974
 975         switch (pbp->bio_cmd) {
 976         case BIO_READ:
 977                 if (pbp->bio_inbed < pbp->bio_children)
 978                         break;
 979                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 1)
 980                         g_io_deliver(pbp, pbp->bio_error);
 981                 else {
 982                         pbp->bio_error = 0;
 983                         mtx_lock(&sc->sc_queue_mtx);
 984                         bioq_disksort(&sc->sc_queue, pbp);
 985                         mtx_unlock(&sc->sc_queue_mtx);
 986                         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 987                         wakeup(sc);
 988                 }
 989                 break;
 990         case BIO_DELETE:
 991         case BIO_WRITE:
 992                 if (pbp->bio_children == 0) {
 993                         /*
 994                          * All requests failed.
 995                          */
 996                 } else if (pbp->bio_inbed < pbp->bio_children) {
 997                         /* Do nothing. */
 998                         break;
 999                 } else if (pbp->bio_children == pbp->bio_inbed) {
1000                         /* Some requests succeeded. */
1001                         pbp->bio_error = 0;
1002                         pbp->bio_completed = pbp->bio_length;
1003                 }
1004                 bioq_remove(&sc->sc_inflight, pbp);
1005                 /* Release delayed sync requests if possible. */
1006                 g_mirror_sync_release(sc);
1007                 g_io_deliver(pbp, pbp->bio_error);
1008                 break;
1009         default:
1010                 KASSERT(1 == 0, ("Invalid request: %u.", pbp->bio_cmd));
1011                 break;
1012         }
1013 }
1014
1015 static void
1016 g_mirror_sync_done(struct bio *bp)
1017 {
1018         struct g_mirror_softc *sc;
1019
1020         G_MIRROR_LOGREQ(3, bp, "Synchronization request delivered.");
1021         sc = bp->bio_from->geom->softc;
1022         bp->bio_cflags = G_MIRROR_BIO_FLAG_SYNC;
1023         mtx_lock(&sc->sc_queue_mtx);
1024         bioq_disksort(&sc->sc_queue, bp);
1025         mtx_unlock(&sc->sc_queue_mtx);
1026         wakeup(sc);
1027 }
1028
1029 static void
1030 g_mirror_candelete(struct bio *bp)
1031 {
1032         struct g_mirror_softc *sc;
1033         struct g_mirror_disk *disk;
1034         int *val;
1035
1036         sc = bp->bio_to->geom->softc;
1037         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1038                 if (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE)
1039                         break;
1040         }
1041         val = (int *)bp->bio_data;
1042         *val = (disk != NULL);
1043         g_io_deliver(bp, 0);
1044 }
1045
1046 static void
1047 g_mirror_kernel_dump(struct bio *bp)
1048 {
1049         struct g_mirror_softc *sc;
1050         struct g_mirror_disk *disk;
1051         struct bio *cbp;
1052         struct g_kerneldump *gkd;
1053
1054         /*
1055          * We configure dumping to the first component, because this component
1056          * will be used for reading with 'prefer' balance algorithm.
1057          * If the component with the higest priority is currently disconnected
1058          * we will not be able to read the dump after the reboot if it will be
1059          * connected and synchronized later. Can we do something better?
1060          */
1061         sc = bp->bio_to->geom->softc;
1062         disk = LIST_FIRST(&sc->sc_disks);
1063
1064         gkd = (struct g_kerneldump *)bp->bio_data;
1065         if (gkd->length > bp->bio_to->mediasize)
1066                 gkd->length = bp->bio_to->mediasize;
1067         cbp = g_clone_bio(bp);
1068         if (cbp == NULL) {
1069                 g_io_deliver(bp, ENOMEM);
1070                 return;
1071         }
1072         cbp->bio_done = g_std_done;
1073         g_io_request(cbp, disk->d_consumer);
1074         G_MIRROR_DEBUG(1, "Kernel dump will go to %s.",
1075             g_mirror_get_diskname(disk));
1076 }
1077
1078 static void
1079 g_mirror_flush(struct g_mirror_softc *sc, struct bio *bp)
1080 {
1081         struct bio_queue_head queue;
1082         struct g_mirror_disk *disk;
1083         struct g_consumer *cp;
1084         struct bio *cbp;
1085
1086         bioq_init(&queue);
1087         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1088                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1089                         continue;
1090                 cbp = g_clone_bio(bp);
1091                 if (cbp == NULL) {
1092                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1093                                 g_destroy_bio(cbp);
1094                         if (bp->bio_error == 0)
1095                                 bp->bio_error = ENOMEM;
1096                         g_io_deliver(bp, bp->bio_error);
1097                         return;
1098                 }
1099                 bioq_insert_tail(&queue, cbp);
1100                 cbp->bio_done = g_mirror_flush_done;
1101                 cbp->bio_caller1 = disk;
1102                 cbp->bio_to = disk->d_consumer->provider;
1103         }
1104         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1105                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1106                 disk = cbp->bio_caller1;
1107                 cbp->bio_caller1 = NULL;
1108                 cp = disk->d_consumer;
1109                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1110                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1111                     cp->acr, cp->acw, cp->ace));
1112                 g_io_request(cbp, disk->d_consumer);
1113         }
1114 }
1115
1116 static void
1117 g_mirror_start(struct bio *bp)
1118 {
1119         struct g_mirror_softc *sc;
1120
1121         sc = bp->bio_to->geom->softc;
1122         /*
1123          * If sc == NULL or there are no valid disks, provider's error
1124          * should be set and g_mirror_start() should not be called at all.
1125          */
1126         KASSERT(sc != NULL && sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1127             ("Provider's error should be set (error=%d)(mirror=%s).",
1128             bp->bio_to->error, bp->bio_to->name));
1129         G_MIRROR_LOGREQ(3, bp, "Request received.");
1130
1131         switch (bp->bio_cmd) {
1132         case BIO_READ:
1133         case BIO_WRITE:
1134         case BIO_DELETE:
1135                 break;
1136         case BIO_FLUSH:
1137                 g_mirror_flush(sc, bp);
1138                 return;
1139         case BIO_GETATTR:
1140                 if (!strcmp(bp->bio_attribute, "GEOM::candelete")) {
1141                         g_mirror_candelete(bp);
1142                         return;
1143                 } else if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) {
1144                         g_mirror_kernel_dump(bp);
1145                         return;
1146                 }
1147                 /* FALLTHROUGH */
1148         default:
1149                 g_io_deliver(bp, EOPNOTSUPP);
1150                 return;
1151         }
1152         mtx_lock(&sc->sc_queue_mtx);
1153         bioq_disksort(&sc->sc_queue, bp);
1154         mtx_unlock(&sc->sc_queue_mtx);
1155         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1156         wakeup(sc);
1157 }
1158
1159 /*
1160  * Return TRUE if the given request is colliding with a in-progress
1161  * synchronization request.
1162  */
1163 static int
1164 g_mirror_sync_collision(struct g_mirror_softc *sc, struct bio *bp)
1165 {
1166         struct g_mirror_disk *disk;
1167         struct bio *sbp;
1168         off_t rstart, rend, sstart, send;
1169         int i;
1170
1171         if (sc->sc_sync.ds_ndisks == 0)
1172                 return (0);
1173         rstart = bp->bio_offset;
1174         rend = bp->bio_offset + bp->bio_length;
1175         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1176                 if (disk->d_state != G_MIRROR_DISK_STATE_SYNCHRONIZING)
1177                         continue;
1178                 for (i = 0; i < g_mirror_syncreqs; i++) {
1179                         sbp = disk->d_sync.ds_bios[i];
1180                         if (sbp == NULL)
1181                                 continue;
1182                         sstart = sbp->bio_offset;
1183                         send = sbp->bio_offset + sbp->bio_length;
1184                         if (rend > sstart && rstart < send)
1185                                 return (1);
1186                 }
1187         }
1188         return (0);
1189 }
1190
1191 /*
1192  * Return TRUE if the given sync request is colliding with a in-progress regular
1193  * request.
1194  */
1195 static int
1196 g_mirror_regular_collision(struct g_mirror_softc *sc, struct bio *sbp)
1197 {
1198         off_t rstart, rend, sstart, send;
1199         struct bio *bp;
1200
1201         if (sc->sc_sync.ds_ndisks == 0)
1202                 return (0);
1203         sstart = sbp->bio_offset;
1204         send = sbp->bio_offset + sbp->bio_length;
1205         TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1206                 rstart = bp->bio_offset;
1207                 rend = bp->bio_offset + bp->bio_length;
1208                 if (rend > sstart && rstart < send)
1209                         return (1);
1210         }
1211         return (0);
1212 }
1213
1214 /*
1215  * Puts request onto delayed queue.
1216  */
1217 static void
1218 g_mirror_regular_delay(struct g_mirror_softc *sc, struct bio *bp)
1219 {
1220
1221         G_MIRROR_LOGREQ(2, bp, "Delaying request.");
1222         bioq_insert_head(&sc->sc_regular_delayed, bp);
1223 }
1224
1225 /*
1226  * Puts synchronization request onto delayed queue.
1227  */
1228 static void
1229 g_mirror_sync_delay(struct g_mirror_softc *sc, struct bio *bp)
1230 {
1231
1232         G_MIRROR_LOGREQ(2, bp, "Delaying synchronization request.");
1233         bioq_insert_tail(&sc->sc_sync_delayed, bp);
1234 }
1235
1236 /*
1237  * Releases delayed regular requests which don't collide anymore with sync
1238  * requests.
1239  */
1240 static void
1241 g_mirror_regular_release(struct g_mirror_softc *sc)
1242 {
1243         struct bio *bp, *bp2;
1244
1245         TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1246                 if (g_mirror_sync_collision(sc, bp))
1247                         continue;
1248                 bioq_remove(&sc->sc_regular_delayed, bp);
1249                 G_MIRROR_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1250                 mtx_lock(&sc->sc_queue_mtx);
1251                 bioq_insert_head(&sc->sc_queue, bp);
1252 #if 0
1253                 /*
1254                  * wakeup() is not needed, because this function is called from
1255                  * the worker thread.
1256                  */
1257                 wakeup(&sc->sc_queue);
1258 #endif
1259                 mtx_unlock(&sc->sc_queue_mtx);
1260         }
1261 }
1262
1263 /*
1264  * Releases delayed sync requests which don't collide anymore with regular
1265  * requests.
1266  */
1267 static void
1268 g_mirror_sync_release(struct g_mirror_softc *sc)
1269 {
1270         struct bio *bp, *bp2;
1271
1272         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1273                 if (g_mirror_regular_collision(sc, bp))
1274                         continue;
1275                 bioq_remove(&sc->sc_sync_delayed, bp);
1276                 G_MIRROR_LOGREQ(2, bp,
1277                     "Releasing delayed synchronization request.");
1278                 g_io_request(bp, bp->bio_from);
1279         }
1280 }
1281
1282 /*
1283  * Handle synchronization requests.
1284  * Every synchronization request is two-steps process: first, READ request is
1285  * send to active provider and then WRITE request (with read data) to the provider
1286  * beeing synchronized. When WRITE is finished, new synchronization request is
1287  * send.
1288  */
1289 static void
1290 g_mirror_sync_request(struct bio *bp)
1291 {
1292         struct g_mirror_softc *sc;
1293         struct g_mirror_disk *disk;
1294
1295         bp->bio_from->index--;
1296         sc = bp->bio_from->geom->softc;
1297         disk = bp->bio_from->private;
1298         if (disk == NULL) {
1299                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1300                 g_topology_lock();
1301                 g_mirror_kill_consumer(sc, bp->bio_from);
1302                 g_topology_unlock();
1303                 free(bp->bio_data, M_MIRROR);
1304                 g_destroy_bio(bp);
1305                 sx_xlock(&sc->sc_lock);
1306                 return;
1307         }
1308
1309         /*
1310          * Synchronization request.
1311          */
1312         switch (bp->bio_cmd) {
1313         case BIO_READ:
1314             {
1315                 struct g_consumer *cp;
1316
1317                 if (bp->bio_error != 0) {
1318                         G_MIRROR_LOGREQ(0, bp,
1319                             "Synchronization request failed (error=%d).",
1320                             bp->bio_error);
1321                         g_destroy_bio(bp);
1322                         return;
1323                 }
1324                 G_MIRROR_LOGREQ(3, bp,
1325                     "Synchronization request half-finished.");
1326                 bp->bio_cmd = BIO_WRITE;
1327                 bp->bio_cflags = 0;
1328                 cp = disk->d_consumer;
1329                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1330                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1331                     cp->acr, cp->acw, cp->ace));
1332                 cp->index++;
1333                 g_io_request(bp, cp);
1334                 return;
1335             }
1336         case BIO_WRITE:
1337             {
1338                 struct g_mirror_disk_sync *sync;
1339                 off_t offset;
1340                 void *data;
1341                 int i;
1342
1343                 if (bp->bio_error != 0) {
1344                         G_MIRROR_LOGREQ(0, bp,
1345                             "Synchronization request failed (error=%d).",
1346                             bp->bio_error);
1347                         g_destroy_bio(bp);
1348                         sc->sc_bump_id |= G_MIRROR_BUMP_GENID;
1349                         g_mirror_event_send(disk,
1350                             G_MIRROR_DISK_STATE_DISCONNECTED,
1351                             G_MIRROR_EVENT_DONTWAIT);
1352                         return;
1353                 }
1354                 G_MIRROR_LOGREQ(3, bp, "Synchronization request finished.");
1355                 sync = &disk->d_sync;
1356                 if (sync->ds_offset >= sc->sc_mediasize ||
1357                     sync->ds_consumer == NULL ||
1358                     (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1359                         /* Don't send more synchronization requests. */
1360                         sync->ds_inflight--;
1361                         if (sync->ds_bios != NULL) {
1362                                 i = (int)(uintptr_t)bp->bio_caller1;
1363                                 sync->ds_bios[i] = NULL;
1364                         }
1365                         free(bp->bio_data, M_MIRROR);
1366                         g_destroy_bio(bp);
1367                         if (sync->ds_inflight > 0)
1368                                 return;
1369                         if (sync->ds_consumer == NULL ||
1370                             (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1371                                 return;
1372                         }
1373                         /* Disk up-to-date, activate it. */
1374                         g_mirror_event_send(disk, G_MIRROR_DISK_STATE_ACTIVE,
1375                             G_MIRROR_EVENT_DONTWAIT);
1376                         return;
1377                 }
1378
1379                 /* Send next synchronization request. */
1380                 data = bp->bio_data;
1381                 bzero(bp, sizeof(*bp));
1382                 bp->bio_cmd = BIO_READ;
1383                 bp->bio_offset = sync->ds_offset;
1384                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1385                 sync->ds_offset += bp->bio_length;
1386                 bp->bio_done = g_mirror_sync_done;
1387                 bp->bio_data = data;
1388                 bp->bio_from = sync->ds_consumer;
1389                 bp->bio_to = sc->sc_provider;
1390                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
1391                 sync->ds_consumer->index++;
1392                 /*
1393                  * Delay the request if it is colliding with a regular request.
1394                  */
1395                 if (g_mirror_regular_collision(sc, bp))
1396                         g_mirror_sync_delay(sc, bp);
1397                 else
1398                         g_io_request(bp, sync->ds_consumer);
1399
1400                 /* Release delayed requests if possible. */
1401                 g_mirror_regular_release(sc);
1402
1403                 /* Find the smallest offset */
1404                 offset = sc->sc_mediasize;
1405                 for (i = 0; i < g_mirror_syncreqs; i++) {
1406                         bp = sync->ds_bios[i];
1407                         if (bp->bio_offset < offset)
1408                                 offset = bp->bio_offset;
1409                 }
1410                 if (sync->ds_offset_done + (MAXPHYS * 100) < offset) {
1411                         /* Update offset_done on every 100 blocks. */
1412                         sync->ds_offset_done = offset;
1413                         g_mirror_update_metadata(disk);
1414                 }
1415                 return;
1416             }
1417         default:
1418                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1419                     bp->bio_cmd, sc->sc_name));
1420                 break;
1421         }
1422 }
1423
1424 static void
1425 g_mirror_request_prefer(struct g_mirror_softc *sc, struct bio *bp)
1426 {
1427         struct g_mirror_disk *disk;
1428         struct g_consumer *cp;
1429         struct bio *cbp;
1430
1431         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1432                 if (disk->d_state == G_MIRROR_DISK_STATE_ACTIVE)
1433                         break;
1434         }
1435         if (disk == NULL) {
1436                 if (bp->bio_error == 0)
1437                         bp->bio_error = ENXIO;
1438                 g_io_deliver(bp, bp->bio_error);
1439                 return;
1440         }
1441         cbp = g_clone_bio(bp);
1442         if (cbp == NULL) {
1443                 if (bp->bio_error == 0)
1444                         bp->bio_error = ENOMEM;
1445                 g_io_deliver(bp, bp->bio_error);
1446                 return;
1447         }
1448         /*
1449          * Fill in the component buf structure.
1450          */
1451         cp = disk->d_consumer;
1452         cbp->bio_done = g_mirror_done;
1453         cbp->bio_to = cp->provider;
1454         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1455         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1456             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1457             cp->acw, cp->ace));
1458         cp->index++;
1459         g_io_request(cbp, cp);
1460 }
1461
1462 static void
1463 g_mirror_request_round_robin(struct g_mirror_softc *sc, struct bio *bp)
1464 {
1465         struct g_mirror_disk *disk;
1466         struct g_consumer *cp;
1467         struct bio *cbp;
1468
1469         disk = g_mirror_get_disk(sc);
1470         if (disk == NULL) {
1471                 if (bp->bio_error == 0)
1472                         bp->bio_error = ENXIO;
1473                 g_io_deliver(bp, bp->bio_error);
1474                 return;
1475         }
1476         cbp = g_clone_bio(bp);
1477         if (cbp == NULL) {
1478                 if (bp->bio_error == 0)
1479                         bp->bio_error = ENOMEM;
1480                 g_io_deliver(bp, bp->bio_error);
1481                 return;
1482         }
1483         /*
1484          * Fill in the component buf structure.
1485          */
1486         cp = disk->d_consumer;
1487         cbp->bio_done = g_mirror_done;
1488         cbp->bio_to = cp->provider;
1489         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1490         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1491             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1492             cp->acw, cp->ace));
1493         cp->index++;
1494         g_io_request(cbp, cp);
1495 }
1496
1497 #define TRACK_SIZE  (1 * 1024 * 1024)
1498 #define LOAD_SCALE      256
1499 #define ABS(x)          (((x) >= 0) ? (x) : (-(x)))
1500
1501 static void
1502 g_mirror_request_load(struct g_mirror_softc *sc, struct bio *bp)
1503 {
1504         struct g_mirror_disk *disk, *dp;
1505         struct g_consumer *cp;
1506         struct bio *cbp;
1507         int prio, best;
1508
1509         /* Find a disk with the smallest load. */
1510         disk = NULL;
1511         best = INT_MAX;
1512         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1513                 if (dp->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1514                         continue;
1515                 prio = dp->load;
1516                 /* If disk head is precisely in position - highly prefer it. */
1517                 if (dp->d_last_offset == bp->bio_offset)
1518                         prio -= 2 * LOAD_SCALE;
1519                 else
1520                 /* If disk head is close to position - prefer it. */
1521                 if (ABS(dp->d_last_offset - bp->bio_offset) < TRACK_SIZE)
1522                         prio -= 1 * LOAD_SCALE;
1523                 if (prio <= best) {
1524                         disk = dp;
1525                         best = prio;
1526                 }
1527         }
1528         KASSERT(disk != NULL, ("NULL disk for %s.", sc->sc_name));
1529         cbp = g_clone_bio(bp);
1530         if (cbp == NULL) {
1531                 if (bp->bio_error == 0)
1532                         bp->bio_error = ENOMEM;
1533                 g_io_deliver(bp, bp->bio_error);
1534                 return;
1535         }
1536         /*
1537          * Fill in the component buf structure.
1538          */
1539         cp = disk->d_consumer;
1540         cbp->bio_done = g_mirror_done;
1541         cbp->bio_to = cp->provider;
1542         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1543         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1544             ("Consumer %s not opened (r%dw%de%d).", cp->provider->name, cp->acr,
1545             cp->acw, cp->ace));
1546         cp->index++;
1547         /* Remember last head position */
1548         disk->d_last_offset = bp->bio_offset + bp->bio_length;
1549         /* Update loads. */
1550         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
1551                 dp->load = (dp->d_consumer->index * LOAD_SCALE +
1552                     dp->load * 7) / 8;
1553         }
1554         g_io_request(cbp, cp);
1555 }
1556
1557 static void
1558 g_mirror_request_split(struct g_mirror_softc *sc, struct bio *bp)
1559 {
1560         struct bio_queue_head queue;
1561         struct g_mirror_disk *disk;
1562         struct g_consumer *cp;
1563         struct bio *cbp;
1564         off_t left, mod, offset, slice;
1565         u_char *data;
1566         u_int ndisks;
1567
1568         if (bp->bio_length <= sc->sc_slice) {
1569                 g_mirror_request_round_robin(sc, bp);
1570                 return;
1571         }
1572         ndisks = g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE);
1573         slice = bp->bio_length / ndisks;
1574         mod = slice % sc->sc_provider->sectorsize;
1575         if (mod != 0)
1576                 slice += sc->sc_provider->sectorsize - mod;
1577         /*
1578          * Allocate all bios before sending any request, so we can
1579          * return ENOMEM in nice and clean way.
1580          */
1581         left = bp->bio_length;
1582         offset = bp->bio_offset;
1583         data = bp->bio_data;
1584         bioq_init(&queue);
1585         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1586                 if (disk->d_state != G_MIRROR_DISK_STATE_ACTIVE)
1587                         continue;
1588                 cbp = g_clone_bio(bp);
1589                 if (cbp == NULL) {
1590                         while ((cbp = bioq_takefirst(&queue)) != NULL)
1591                                 bioq_remove(&queue, cbp);
1592                         if (bp->bio_error == 0)
1593                                 bp->bio_error = ENOMEM;
1594                         g_io_deliver(bp, bp->bio_error);
1595                         return;
1596                 }
1597                 bioq_insert_tail(&queue, cbp);
1598                 cbp->bio_done = g_mirror_done;
1599                 cbp->bio_caller1 = disk;
1600                 cbp->bio_to = disk->d_consumer->provider;
1601                 cbp->bio_offset = offset;
1602                 cbp->bio_data = data;
1603                 cbp->bio_length = MIN(left, slice);
1604                 left -= cbp->bio_length;
1605                 if (left == 0)
1606                         break;
1607                 offset += cbp->bio_length;
1608                 data += cbp->bio_length;
1609         }
1610         while ((cbp = bioq_takefirst(&queue)) != NULL) {
1611                 G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1612                 disk = cbp->bio_caller1;
1613                 cbp->bio_caller1 = NULL;
1614                 cp = disk->d_consumer;
1615                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1616                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1617                     cp->acr, cp->acw, cp->ace));
1618                 disk->d_consumer->index++;
1619                 g_io_request(cbp, disk->d_consumer);
1620         }
1621 }
1622
1623 static void
1624 g_mirror_register_request(struct bio *bp)
1625 {
1626         struct g_mirror_softc *sc;
1627
1628         sc = bp->bio_to->geom->softc;
1629         switch (bp->bio_cmd) {
1630         case BIO_READ:
1631                 switch (sc->sc_balance) {
1632                 case G_MIRROR_BALANCE_LOAD:
1633                         g_mirror_request_load(sc, bp);
1634                         break;
1635                 case G_MIRROR_BALANCE_PREFER:
1636                         g_mirror_request_prefer(sc, bp);
1637                         break;
1638                 case G_MIRROR_BALANCE_ROUND_ROBIN:
1639                         g_mirror_request_round_robin(sc, bp);
1640                         break;
1641                 case G_MIRROR_BALANCE_SPLIT:
1642                         g_mirror_request_split(sc, bp);
1643                         break;
1644                 }
1645                 return;
1646         case BIO_WRITE:
1647         case BIO_DELETE:
1648             {
1649                 struct g_mirror_disk *disk;
1650                 struct g_mirror_disk_sync *sync;
1651                 struct bio_queue_head queue;
1652                 struct g_consumer *cp;
1653                 struct bio *cbp;
1654
1655                 /*
1656                  * Delay the request if it is colliding with a synchronization
1657                  * request.
1658                  */
1659                 if (g_mirror_sync_collision(sc, bp)) {
1660                         g_mirror_regular_delay(sc, bp);
1661                         return;
1662                 }
1663
1664                 if (sc->sc_idle)
1665                         g_mirror_unidle(sc);
1666                 else
1667                         sc->sc_last_write = time_uptime;
1668
1669                 /*
1670                  * Allocate all bios before sending any request, so we can
1671                  * return ENOMEM in nice and clean way.
1672                  */
1673                 bioq_init(&queue);
1674                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
1675                         sync = &disk->d_sync;
1676                         switch (disk->d_state) {
1677                         case G_MIRROR_DISK_STATE_ACTIVE:
1678                                 break;
1679                         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
1680                                 if (bp->bio_offset >= sync->ds_offset)
1681                                         continue;
1682                                 break;
1683                         default:
1684                                 continue;
1685                         }
1686                         if (bp->bio_cmd == BIO_DELETE &&
1687                             (disk->d_flags & G_MIRROR_DISK_FLAG_CANDELETE) == 0)
1688                                 continue;
1689                         cbp = g_clone_bio(bp);
1690                         if (cbp == NULL) {
1691                                 while ((cbp = bioq_takefirst(&queue)) != NULL)
1692                                         g_destroy_bio(cbp);
1693                                 if (bp->bio_error == 0)
1694                                         bp->bio_error = ENOMEM;
1695                                 g_io_deliver(bp, bp->bio_error);
1696                                 return;
1697                         }
1698                         bioq_insert_tail(&queue, cbp);
1699                         cbp->bio_done = g_mirror_done;
1700                         cp = disk->d_consumer;
1701                         cbp->bio_caller1 = cp;
1702                         cbp->bio_to = cp->provider;
1703                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1704                             ("Consumer %s not opened (r%dw%de%d).",
1705                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1706                 }
1707                 if (bioq_first(&queue) == NULL) {
1708                         g_io_deliver(bp, EOPNOTSUPP);
1709                         return;
1710                 }
1711                 while ((cbp = bioq_takefirst(&queue)) != NULL) {
1712                         G_MIRROR_LOGREQ(3, cbp, "Sending request.");
1713                         cp = cbp->bio_caller1;
1714                         cbp->bio_caller1 = NULL;
1715                         cp->index++;
1716                         sc->sc_writes++;
1717                         g_io_request(cbp, cp);
1718                 }
1719                 /*
1720                  * Put request onto inflight queue, so we can check if new
1721                  * synchronization requests don't collide with it.
1722                  */
1723                 bioq_insert_tail(&sc->sc_inflight, bp);
1724                 /*
1725                  * Bump syncid on first write.
1726                  */
1727                 if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0) {
1728                         sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
1729                         g_mirror_bump_syncid(sc);
1730                 }
1731                 return;
1732             }
1733         default:
1734                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1735                     bp->bio_cmd, sc->sc_name));
1736                 break;
1737         }
1738 }
1739
1740 static int
1741 g_mirror_can_destroy(struct g_mirror_softc *sc)
1742 {
1743         struct g_geom *gp;
1744         struct g_consumer *cp;
1745
1746         g_topology_assert();
1747         gp = sc->sc_geom;
1748         if (gp->softc == NULL)
1749                 return (1);
1750         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_TASTING) != 0)
1751                 return (0);
1752         LIST_FOREACH(cp, &gp->consumer, consumer) {
1753                 if (g_mirror_is_busy(sc, cp))
1754                         return (0);
1755         }
1756         gp = sc->sc_sync.ds_geom;
1757         LIST_FOREACH(cp, &gp->consumer, consumer) {
1758                 if (g_mirror_is_busy(sc, cp))
1759                         return (0);
1760         }
1761         G_MIRROR_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1762             sc->sc_name);
1763         return (1);
1764 }
1765
1766 static int
1767 g_mirror_try_destroy(struct g_mirror_softc *sc)
1768 {
1769
1770         if (sc->sc_rootmount != NULL) {
1771                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1772                     sc->sc_rootmount);
1773                 root_mount_rel(sc->sc_rootmount);
1774                 sc->sc_rootmount = NULL;
1775         }
1776         g_topology_lock();
1777         if (!g_mirror_can_destroy(sc)) {
1778                 g_topology_unlock();
1779                 return (0);
1780         }
1781         sc->sc_geom->softc = NULL;
1782         sc->sc_sync.ds_geom->softc = NULL;
1783         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_WAIT) != 0) {
1784                 g_topology_unlock();
1785                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1786                     &sc->sc_worker);
1787                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
1788                 sx_xunlock(&sc->sc_lock);
1789                 wakeup(&sc->sc_worker);
1790                 sc->sc_worker = NULL;
1791         } else {
1792                 g_topology_unlock();
1793                 g_mirror_destroy_device(sc);
1794                 free(sc, M_MIRROR);
1795         }
1796         return (1);
1797 }
1798
1799 /*
1800  * Worker thread.
1801  */
1802 static void
1803 g_mirror_worker(void *arg)
1804 {
1805         struct g_mirror_softc *sc;
1806         struct g_mirror_event *ep;
1807         struct bio *bp;
1808         int timeout;
1809
1810         sc = arg;
1811         thread_lock(curthread);
1812         sched_prio(curthread, PRIBIO);
1813         thread_unlock(curthread);
1814
1815         sx_xlock(&sc->sc_lock);
1816         for (;;) {
1817                 G_MIRROR_DEBUG(5, "%s: Let's see...", __func__);
1818                 /*
1819                  * First take a look at events.
1820                  * This is important to handle events before any I/O requests.
1821                  */
1822                 ep = g_mirror_event_get(sc);
1823                 if (ep != NULL) {
1824                         g_mirror_event_remove(sc, ep);
1825                         if ((ep->e_flags & G_MIRROR_EVENT_DEVICE) != 0) {
1826                                 /* Update only device status. */
1827                                 G_MIRROR_DEBUG(3,
1828                                     "Running event for device %s.",
1829                                     sc->sc_name);
1830                                 ep->e_error = 0;
1831                                 g_mirror_update_device(sc, 1);
1832                         } else {
1833                                 /* Update disk status. */
1834                                 G_MIRROR_DEBUG(3, "Running event for disk %s.",
1835                                      g_mirror_get_diskname(ep->e_disk));
1836                                 ep->e_error = g_mirror_update_disk(ep->e_disk,
1837                                     ep->e_state);
1838                                 if (ep->e_error == 0)
1839                                         g_mirror_update_device(sc, 0);
1840                         }
1841                         if ((ep->e_flags & G_MIRROR_EVENT_DONTWAIT) != 0) {
1842                                 KASSERT(ep->e_error == 0,
1843                                     ("Error cannot be handled."));
1844                                 g_mirror_event_free(ep);
1845                         } else {
1846                                 ep->e_flags |= G_MIRROR_EVENT_DONE;
1847                                 G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__,
1848                                     ep);
1849                                 mtx_lock(&sc->sc_events_mtx);
1850                                 wakeup(ep);
1851                                 mtx_unlock(&sc->sc_events_mtx);
1852                         }
1853                         if ((sc->sc_flags &
1854                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1855                                 if (g_mirror_try_destroy(sc)) {
1856                                         curthread->td_pflags &= ~TDP_GEOM;
1857                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1858                                         kproc_exit(0);
1859                                 }
1860                         }
1861                         G_MIRROR_DEBUG(5, "%s: I'm here 1.", __func__);
1862                         continue;
1863                 }
1864                 /*
1865                  * Check if we can mark array as CLEAN and if we can't take
1866                  * how much seconds should we wait.
1867                  */
1868                 timeout = g_mirror_idle(sc, -1);
1869                 /*
1870                  * Now I/O requests.
1871                  */
1872                 /* Get first request from the queue. */
1873                 mtx_lock(&sc->sc_queue_mtx);
1874                 bp = bioq_first(&sc->sc_queue);
1875                 if (bp == NULL) {
1876                         if ((sc->sc_flags &
1877                             G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
1878                                 mtx_unlock(&sc->sc_queue_mtx);
1879                                 if (g_mirror_try_destroy(sc)) {
1880                                         curthread->td_pflags &= ~TDP_GEOM;
1881                                         G_MIRROR_DEBUG(1, "Thread exiting.");
1882                                         kproc_exit(0);
1883                                 }
1884                                 mtx_lock(&sc->sc_queue_mtx);
1885                         }
1886                         sx_xunlock(&sc->sc_lock);
1887                         /*
1888                          * XXX: We can miss an event here, because an event
1889                          *      can be added without sx-device-lock and without
1890                          *      mtx-queue-lock. Maybe I should just stop using
1891                          *      dedicated mutex for events synchronization and
1892                          *      stick with the queue lock?
1893                          *      The event will hang here until next I/O request
1894                          *      or next event is received.
1895                          */
1896                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "m:w1",
1897                             timeout * hz);
1898                         sx_xlock(&sc->sc_lock);
1899                         G_MIRROR_DEBUG(5, "%s: I'm here 4.", __func__);
1900                         continue;
1901                 }
1902                 bioq_remove(&sc->sc_queue, bp);
1903                 mtx_unlock(&sc->sc_queue_mtx);
1904
1905                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
1906                     (bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0) {
1907                         g_mirror_sync_request(bp);      /* READ */
1908                 } else if (bp->bio_to != sc->sc_provider) {
1909                         if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_REGULAR) != 0)
1910                                 g_mirror_regular_request(bp);
1911                         else if ((bp->bio_cflags & G_MIRROR_BIO_FLAG_SYNC) != 0)
1912                                 g_mirror_sync_request(bp);      /* WRITE */
1913                         else {
1914                                 KASSERT(0,
1915                                     ("Invalid request cflags=0x%hhx to=%s.",
1916                                     bp->bio_cflags, bp->bio_to->name));
1917                         }
1918                 } else {
1919                         g_mirror_register_request(bp);
1920                 }
1921                 G_MIRROR_DEBUG(5, "%s: I'm here 9.", __func__);
1922         }
1923 }
1924
1925 static void
1926 g_mirror_update_idle(struct g_mirror_softc *sc, struct g_mirror_disk *disk)
1927 {
1928
1929         sx_assert(&sc->sc_lock, SX_LOCKED);
1930
1931         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) != 0)
1932                 return;
1933         if (!sc->sc_idle && (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) == 0) {
1934                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as dirty.",
1935                     g_mirror_get_diskname(disk), sc->sc_name);
1936                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1937         } else if (sc->sc_idle &&
1938             (disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
1939                 G_MIRROR_DEBUG(1, "Disk %s (device %s) marked as clean.",
1940                     g_mirror_get_diskname(disk), sc->sc_name);
1941                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
1942         }
1943 }
1944
1945 static void
1946 g_mirror_sync_start(struct g_mirror_disk *disk)
1947 {
1948         struct g_mirror_softc *sc;
1949         struct g_consumer *cp;
1950         struct bio *bp;
1951         int error, i;
1952
1953         g_topology_assert_not();
1954         sc = disk->d_softc;
1955         sx_assert(&sc->sc_lock, SX_LOCKED);
1956
1957         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
1958             ("Disk %s is not marked for synchronization.",
1959             g_mirror_get_diskname(disk)));
1960         KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
1961             ("Device not in RUNNING state (%s, %u).", sc->sc_name,
1962             sc->sc_state));
1963
1964         sx_xunlock(&sc->sc_lock);
1965         g_topology_lock();
1966         cp = g_new_consumer(sc->sc_sync.ds_geom);
1967         cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
1968         error = g_attach(cp, sc->sc_provider);
1969         KASSERT(error == 0,
1970             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
1971         error = g_access(cp, 1, 0, 0);
1972         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
1973         g_topology_unlock();
1974         sx_xlock(&sc->sc_lock);
1975
1976         G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
1977             g_mirror_get_diskname(disk));
1978         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOFAILSYNC) == 0)
1979                 disk->d_flags |= G_MIRROR_DISK_FLAG_DIRTY;
1980         KASSERT(disk->d_sync.ds_consumer == NULL,
1981             ("Sync consumer already exists (device=%s, disk=%s).",
1982             sc->sc_name, g_mirror_get_diskname(disk)));
1983
1984         disk->d_sync.ds_consumer = cp;
1985         disk->d_sync.ds_consumer->private = disk;
1986         disk->d_sync.ds_consumer->index = 0;
1987
1988         /*
1989          * Allocate memory for synchronization bios and initialize them.
1990          */
1991         disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_mirror_syncreqs,
1992             M_MIRROR, M_WAITOK);
1993         for (i = 0; i < g_mirror_syncreqs; i++) {
1994                 bp = g_alloc_bio();
1995                 disk->d_sync.ds_bios[i] = bp;
1996                 bp->bio_parent = NULL;
1997                 bp->bio_cmd = BIO_READ;
1998                 bp->bio_data = malloc(MAXPHYS, M_MIRROR, M_WAITOK);
1999                 bp->bio_cflags = 0;
2000                 bp->bio_offset = disk->d_sync.ds_offset;
2001                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
2002                 disk->d_sync.ds_offset += bp->bio_length;
2003                 bp->bio_done = g_mirror_sync_done;
2004                 bp->bio_from = disk->d_sync.ds_consumer;
2005                 bp->bio_to = sc->sc_provider;
2006                 bp->bio_caller1 = (void *)(uintptr_t)i;
2007         }
2008
2009         /* Increase the number of disks in SYNCHRONIZING state. */
2010         sc->sc_sync.ds_ndisks++;
2011         /* Set the number of in-flight synchronization requests. */
2012         disk->d_sync.ds_inflight = g_mirror_syncreqs;
2013
2014         /*
2015          * Fire off first synchronization requests.
2016          */
2017         for (i = 0; i < g_mirror_syncreqs; i++) {
2018                 bp = disk->d_sync.ds_bios[i];
2019                 G_MIRROR_LOGREQ(3, bp, "Sending synchronization request.");
2020                 disk->d_sync.ds_consumer->index++;
2021                 /*
2022                  * Delay the request if it is colliding with a regular request.
2023                  */
2024                 if (g_mirror_regular_collision(sc, bp))
2025                         g_mirror_sync_delay(sc, bp);
2026                 else
2027                         g_io_request(bp, disk->d_sync.ds_consumer);
2028         }
2029 }
2030
2031 /*
2032  * Stop synchronization process.
2033  * type: 0 - synchronization finished
2034  *       1 - synchronization stopped
2035  */
2036 static void
2037 g_mirror_sync_stop(struct g_mirror_disk *disk, int type)
2038 {
2039         struct g_mirror_softc *sc;
2040         struct g_consumer *cp;
2041
2042         g_topology_assert_not();
2043         sc = disk->d_softc;
2044         sx_assert(&sc->sc_lock, SX_LOCKED);
2045
2046         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2047             ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2048             g_mirror_disk_state2str(disk->d_state)));
2049         if (disk->d_sync.ds_consumer == NULL)
2050                 return;
2051
2052         if (type == 0) {
2053                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2054                     sc->sc_name, g_mirror_get_diskname(disk));
2055         } else /* if (type == 1) */ {
2056                 G_MIRROR_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2057                     sc->sc_name, g_mirror_get_diskname(disk));
2058         }
2059         free(disk->d_sync.ds_bios, M_MIRROR);
2060         disk->d_sync.ds_bios = NULL;
2061         cp = disk->d_sync.ds_consumer;
2062         disk->d_sync.ds_consumer = NULL;
2063         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2064         sc->sc_sync.ds_ndisks--;
2065         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2066         g_topology_lock();
2067         g_mirror_kill_consumer(sc, cp);
2068         g_topology_unlock();
2069         sx_xlock(&sc->sc_lock);
2070 }
2071
2072 static void
2073 g_mirror_launch_provider(struct g_mirror_softc *sc)
2074 {
2075         struct g_mirror_disk *disk;
2076         struct g_provider *pp, *dp;
2077
2078         sx_assert(&sc->sc_lock, SX_LOCKED);
2079
2080         g_topology_lock();
2081         pp = g_new_providerf(sc->sc_geom, "mirror/%s", sc->sc_name);
2082         pp->flags |= G_PF_DIRECT_RECEIVE;
2083         pp->mediasize = sc->sc_mediasize;
2084         pp->sectorsize = sc->sc_sectorsize;
2085         pp->stripesize = 0;
2086         pp->stripeoffset = 0;
2087
2088         /* Splitting of unmapped BIO's could work but isn't implemented now */
2089         if (sc->sc_balance != G_MIRROR_BALANCE_SPLIT)
2090                 pp->flags |= G_PF_ACCEPT_UNMAPPED;
2091
2092         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2093                 if (disk->d_consumer && disk->d_consumer->provider) {
2094                         dp = disk->d_consumer->provider;
2095                         if (dp->stripesize > pp->stripesize) {
2096                                 pp->stripesize = dp->stripesize;
2097                                 pp->stripeoffset = dp->stripeoffset;
2098                         }
2099                         /* A provider underneath us doesn't support unmapped */
2100                         if ((dp->flags & G_PF_ACCEPT_UNMAPPED) == 0) {
2101                                 G_MIRROR_DEBUG(0, "Cancelling unmapped "
2102                                     "because of %s.", dp->name);
2103                                 pp->flags &= ~G_PF_ACCEPT_UNMAPPED;
2104                         }
2105                 }
2106         }
2107         sc->sc_provider = pp;
2108         g_error_provider(pp, 0);
2109         g_topology_unlock();
2110         G_MIRROR_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2111             g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE), sc->sc_ndisks);
2112         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2113                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2114                         g_mirror_sync_start(disk);
2115         }
2116 }
2117
2118 static void
2119 g_mirror_destroy_provider(struct g_mirror_softc *sc)
2120 {
2121         struct g_mirror_disk *disk;
2122         struct bio *bp;
2123
2124         g_topology_assert_not();
2125         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2126             sc->sc_name));
2127
2128         g_topology_lock();
2129         g_error_provider(sc->sc_provider, ENXIO);
2130         mtx_lock(&sc->sc_queue_mtx);
2131         while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL)
2132                 g_io_deliver(bp, ENXIO);
2133         mtx_unlock(&sc->sc_queue_mtx);
2134         G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2135             sc->sc_provider->name);
2136         sc->sc_provider->flags |= G_PF_WITHER;
2137         g_orphan_provider(sc->sc_provider, ENXIO);
2138         g_topology_unlock();
2139         sc->sc_provider = NULL;
2140         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2141                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING)
2142                         g_mirror_sync_stop(disk, 1);
2143         }
2144 }
2145
2146 static void
2147 g_mirror_go(void *arg)
2148 {
2149         struct g_mirror_softc *sc;
2150
2151         sc = arg;
2152         G_MIRROR_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2153         g_mirror_event_send(sc, 0,
2154             G_MIRROR_EVENT_DONTWAIT | G_MIRROR_EVENT_DEVICE);
2155 }
2156
2157 static u_int
2158 g_mirror_determine_state(struct g_mirror_disk *disk)
2159 {
2160         struct g_mirror_softc *sc;
2161         u_int state;
2162
2163         sc = disk->d_softc;
2164         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2165                 if ((disk->d_flags &
2166                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2167                         /* Disk does not need synchronization. */
2168                         state = G_MIRROR_DISK_STATE_ACTIVE;
2169                 } else {
2170                         if ((sc->sc_flags &
2171                              G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2172                             (disk->d_flags &
2173                              G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2174                                 /*
2175                                  * We can start synchronization from
2176                                  * the stored offset.
2177                                  */
2178                                 state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2179                         } else {
2180                                 state = G_MIRROR_DISK_STATE_STALE;
2181                         }
2182                 }
2183         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2184                 /*
2185                  * Reset all synchronization data for this disk,
2186                  * because if it even was synchronized, it was
2187                  * synchronized to disks with different syncid.
2188                  */
2189                 disk->d_flags |= G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2190                 disk->d_sync.ds_offset = 0;
2191                 disk->d_sync.ds_offset_done = 0;
2192                 disk->d_sync.ds_syncid = sc->sc_syncid;
2193                 if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2194                     (disk->d_flags & G_MIRROR_DISK_FLAG_FORCE_SYNC) != 0) {
2195                         state = G_MIRROR_DISK_STATE_SYNCHRONIZING;
2196                 } else {
2197                         state = G_MIRROR_DISK_STATE_STALE;
2198                 }
2199         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2200                 /*
2201                  * Not good, NOT GOOD!
2202                  * It means that mirror was started on stale disks
2203                  * and more fresh disk just arrive.
2204                  * If there were writes, mirror is broken, sorry.
2205                  * I think the best choice here is don't touch
2206                  * this disk and inform the user loudly.
2207                  */
2208                 G_MIRROR_DEBUG(0, "Device %s was started before the freshest "
2209                     "disk (%s) arrives!! It will not be connected to the "
2210                     "running device.", sc->sc_name,
2211                     g_mirror_get_diskname(disk));
2212                 g_mirror_destroy_disk(disk);
2213                 state = G_MIRROR_DISK_STATE_NONE;
2214                 /* Return immediately, because disk was destroyed. */
2215                 return (state);
2216         }
2217         G_MIRROR_DEBUG(3, "State for %s disk: %s.",
2218             g_mirror_get_diskname(disk), g_mirror_disk_state2str(state));
2219         return (state);
2220 }
2221
2222 /*
2223  * Update device state.
2224  */
2225 static void
2226 g_mirror_update_device(struct g_mirror_softc *sc, boolean_t force)
2227 {
2228         struct g_mirror_disk *disk;
2229         u_int state;
2230
2231         sx_assert(&sc->sc_lock, SX_XLOCKED);
2232
2233         switch (sc->sc_state) {
2234         case G_MIRROR_DEVICE_STATE_STARTING:
2235             {
2236                 struct g_mirror_disk *pdisk, *tdisk;
2237                 u_int dirty, ndisks, genid, syncid;
2238
2239                 KASSERT(sc->sc_provider == NULL,
2240                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2241                 /*
2242                  * Are we ready? We are, if all disks are connected or
2243                  * if we have any disks and 'force' is true.
2244                  */
2245                 ndisks = g_mirror_ndisks(sc, -1);
2246                 if (sc->sc_ndisks == ndisks || (force && ndisks > 0)) {
2247                         ;
2248                 } else if (ndisks == 0) {
2249                         /*
2250                          * Disks went down in starting phase, so destroy
2251                          * device.
2252                          */
2253                         callout_drain(&sc->sc_callout);
2254                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2255                         G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2256                             sc->sc_rootmount);
2257                         root_mount_rel(sc->sc_rootmount);
2258                         sc->sc_rootmount = NULL;
2259                         return;
2260                 } else {
2261                         return;
2262                 }
2263
2264                 /*
2265                  * Activate all disks with the biggest syncid.
2266                  */
2267                 if (force) {
2268                         /*
2269                          * If 'force' is true, we have been called due to
2270                          * timeout, so don't bother canceling timeout.
2271                          */
2272                         ndisks = 0;
2273                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2274                                 if ((disk->d_flags &
2275                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) == 0) {
2276                                         ndisks++;
2277                                 }
2278                         }
2279                         if (ndisks == 0) {
2280                                 /* No valid disks found, destroy device. */
2281                                 sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2282                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2283                                     __LINE__, sc->sc_rootmount);
2284                                 root_mount_rel(sc->sc_rootmount);
2285                                 sc->sc_rootmount = NULL;
2286                                 return;
2287                         }
2288                 } else {
2289                         /* Cancel timeout. */
2290                         callout_drain(&sc->sc_callout);
2291                 }
2292
2293                 /*
2294                  * Find the biggest genid.
2295                  */
2296                 genid = 0;
2297                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2298                         if (disk->d_genid > genid)
2299                                 genid = disk->d_genid;
2300                 }
2301                 sc->sc_genid = genid;
2302                 /*
2303                  * Remove all disks without the biggest genid.
2304                  */
2305                 LIST_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
2306                         if (disk->d_genid < genid) {
2307                                 G_MIRROR_DEBUG(0,
2308                                     "Component %s (device %s) broken, skipping.",
2309                                     g_mirror_get_diskname(disk), sc->sc_name);
2310                                 g_mirror_destroy_disk(disk);
2311                         }
2312                 }
2313
2314                 /*
2315                  * Find the biggest syncid.
2316                  */
2317                 syncid = 0;
2318                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2319                         if (disk->d_sync.ds_syncid > syncid)
2320                                 syncid = disk->d_sync.ds_syncid;
2321                 }
2322
2323                 /*
2324                  * Here we need to look for dirty disks and if all disks
2325                  * with the biggest syncid are dirty, we have to choose
2326                  * one with the biggest priority and rebuild the rest.
2327                  */
2328                 /*
2329                  * Find the number of dirty disks with the biggest syncid.
2330                  * Find the number of disks with the biggest syncid.
2331                  * While here, find a disk with the biggest priority.
2332                  */
2333                 dirty = ndisks = 0;
2334                 pdisk = NULL;
2335                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2336                         if (disk->d_sync.ds_syncid != syncid)
2337                                 continue;
2338                         if ((disk->d_flags &
2339                             G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2340                                 continue;
2341                         }
2342                         ndisks++;
2343                         if ((disk->d_flags & G_MIRROR_DISK_FLAG_DIRTY) != 0) {
2344                                 dirty++;
2345                                 if (pdisk == NULL ||
2346                                     pdisk->d_priority < disk->d_priority) {
2347                                         pdisk = disk;
2348                                 }
2349                         }
2350                 }
2351                 if (dirty == 0) {
2352                         /* No dirty disks at all, great. */
2353                 } else if (dirty == ndisks) {
2354                         /*
2355                          * Force synchronization for all dirty disks except one
2356                          * with the biggest priority.
2357                          */
2358                         KASSERT(pdisk != NULL, ("pdisk == NULL"));
2359                         G_MIRROR_DEBUG(1, "Using disk %s (device %s) as a "
2360                             "master disk for synchronization.",
2361                             g_mirror_get_diskname(pdisk), sc->sc_name);
2362                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2363                                 if (disk->d_sync.ds_syncid != syncid)
2364                                         continue;
2365                                 if ((disk->d_flags &
2366                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2367                                         continue;
2368                                 }
2369                                 KASSERT((disk->d_flags &
2370                                     G_MIRROR_DISK_FLAG_DIRTY) != 0,
2371                                     ("Disk %s isn't marked as dirty.",
2372                                     g_mirror_get_diskname(disk)));
2373                                 /* Skip the disk with the biggest priority. */
2374                                 if (disk == pdisk)
2375                                         continue;
2376                                 disk->d_sync.ds_syncid = 0;
2377                         }
2378                 } else if (dirty < ndisks) {
2379                         /*
2380                          * Force synchronization for all dirty disks.
2381                          * We have some non-dirty disks.
2382                          */
2383                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2384                                 if (disk->d_sync.ds_syncid != syncid)
2385                                         continue;
2386                                 if ((disk->d_flags &
2387                                     G_MIRROR_DISK_FLAG_SYNCHRONIZING) != 0) {
2388                                         continue;
2389                                 }
2390                                 if ((disk->d_flags &
2391                                     G_MIRROR_DISK_FLAG_DIRTY) == 0) {
2392                                         continue;
2393                                 }
2394                                 disk->d_sync.ds_syncid = 0;
2395                         }
2396                 }
2397
2398                 /* Reset hint. */
2399                 sc->sc_hint = NULL;
2400                 sc->sc_syncid = syncid;
2401                 if (force) {
2402                         /* Remember to bump syncid on first write. */
2403                         sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2404                 }
2405                 state = G_MIRROR_DEVICE_STATE_RUNNING;
2406                 G_MIRROR_DEBUG(1, "Device %s state changed from %s to %s.",
2407                     sc->sc_name, g_mirror_device_state2str(sc->sc_state),
2408                     g_mirror_device_state2str(state));
2409                 sc->sc_state = state;
2410                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2411                         state = g_mirror_determine_state(disk);
2412                         g_mirror_event_send(disk, state,
2413                             G_MIRROR_EVENT_DONTWAIT);
2414                         if (state == G_MIRROR_DISK_STATE_STALE)
2415                                 sc->sc_bump_id |= G_MIRROR_BUMP_SYNCID;
2416                 }
2417                 break;
2418             }
2419         case G_MIRROR_DEVICE_STATE_RUNNING:
2420                 if (g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE) == 0 &&
2421                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2422                         /*
2423                          * No active disks or no disks at all,
2424                          * so destroy device.
2425                          */
2426                         if (sc->sc_provider != NULL)
2427                                 g_mirror_destroy_provider(sc);
2428                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
2429                         break;
2430                 } else if (g_mirror_ndisks(sc,
2431                     G_MIRROR_DISK_STATE_ACTIVE) > 0 &&
2432                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_NEW) == 0) {
2433                         /*
2434                          * We have active disks, launch provider if it doesn't
2435                          * exist.
2436                          */
2437                         if (sc->sc_provider == NULL)
2438                                 g_mirror_launch_provider(sc);
2439                         if (sc->sc_rootmount != NULL) {
2440                                 G_MIRROR_DEBUG(1, "root_mount_rel[%u] %p",
2441                                     __LINE__, sc->sc_rootmount);
2442                                 root_mount_rel(sc->sc_rootmount);
2443                                 sc->sc_rootmount = NULL;
2444                         }
2445                 }
2446                 /*
2447                  * Genid should be bumped immediately, so do it here.
2448                  */
2449                 if ((sc->sc_bump_id & G_MIRROR_BUMP_GENID) != 0) {
2450                         sc->sc_bump_id &= ~G_MIRROR_BUMP_GENID;
2451                         g_mirror_bump_genid(sc);
2452                 }
2453                 break;
2454         default:
2455                 KASSERT(1 == 0, ("Wrong device state (%s, %s).",
2456                     sc->sc_name, g_mirror_device_state2str(sc->sc_state)));
2457                 break;
2458         }
2459 }
2460
2461 /*
2462  * Update disk state and device state if needed.
2463  */
2464 #define DISK_STATE_CHANGED()    G_MIRROR_DEBUG(1,                       \
2465         "Disk %s state changed from %s to %s (device %s).",             \
2466         g_mirror_get_diskname(disk),                                    \
2467         g_mirror_disk_state2str(disk->d_state),                         \
2468         g_mirror_disk_state2str(state), sc->sc_name)
2469 static int
2470 g_mirror_update_disk(struct g_mirror_disk *disk, u_int state)
2471 {
2472         struct g_mirror_softc *sc;
2473
2474         sc = disk->d_softc;
2475         sx_assert(&sc->sc_lock, SX_XLOCKED);
2476
2477 again:
2478         G_MIRROR_DEBUG(3, "Changing disk %s state from %s to %s.",
2479             g_mirror_get_diskname(disk), g_mirror_disk_state2str(disk->d_state),
2480             g_mirror_disk_state2str(state));
2481         switch (state) {
2482         case G_MIRROR_DISK_STATE_NEW:
2483                 /*
2484                  * Possible scenarios:
2485                  * 1. New disk arrive.
2486                  */
2487                 /* Previous state should be NONE. */
2488                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NONE,
2489                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2490                     g_mirror_disk_state2str(disk->d_state)));
2491                 DISK_STATE_CHANGED();
2492
2493                 disk->d_state = state;
2494                 if (LIST_EMPTY(&sc->sc_disks))
2495                         LIST_INSERT_HEAD(&sc->sc_disks, disk, d_next);
2496                 else {
2497                         struct g_mirror_disk *dp;
2498
2499                         LIST_FOREACH(dp, &sc->sc_disks, d_next) {
2500                                 if (disk->d_priority >= dp->d_priority) {
2501                                         LIST_INSERT_BEFORE(dp, disk, d_next);
2502                                         dp = NULL;
2503                                         break;
2504                                 }
2505                                 if (LIST_NEXT(dp, d_next) == NULL)
2506                                         break;
2507                         }
2508                         if (dp != NULL)
2509                                 LIST_INSERT_AFTER(dp, disk, d_next);
2510                 }
2511                 G_MIRROR_DEBUG(1, "Device %s: provider %s detected.",
2512                     sc->sc_name, g_mirror_get_diskname(disk));
2513                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
2514                         break;
2515                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2516                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2517                     g_mirror_device_state2str(sc->sc_state),
2518                     g_mirror_get_diskname(disk),
2519                     g_mirror_disk_state2str(disk->d_state)));
2520                 state = g_mirror_determine_state(disk);
2521                 if (state != G_MIRROR_DISK_STATE_NONE)
2522                         goto again;
2523                 break;
2524         case G_MIRROR_DISK_STATE_ACTIVE:
2525                 /*
2526                  * Possible scenarios:
2527                  * 1. New disk does not need synchronization.
2528                  * 2. Synchronization process finished successfully.
2529                  */
2530                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2531                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2532                     g_mirror_device_state2str(sc->sc_state),
2533                     g_mirror_get_diskname(disk),
2534                     g_mirror_disk_state2str(disk->d_state)));
2535                 /* Previous state should be NEW or SYNCHRONIZING. */
2536                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW ||
2537                     disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2538                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2539                     g_mirror_disk_state2str(disk->d_state)));
2540                 DISK_STATE_CHANGED();
2541
2542                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
2543                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_SYNCHRONIZING;
2544                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_FORCE_SYNC;
2545                         g_mirror_sync_stop(disk, 0);
2546                 }
2547                 disk->d_state = state;
2548                 disk->d_sync.ds_offset = 0;
2549                 disk->d_sync.ds_offset_done = 0;
2550                 g_mirror_update_idle(sc, disk);
2551                 g_mirror_update_metadata(disk);
2552                 G_MIRROR_DEBUG(1, "Device %s: provider %s activated.",
2553                     sc->sc_name, g_mirror_get_diskname(disk));
2554                 break;
2555         case G_MIRROR_DISK_STATE_STALE:
2556                 /*
2557                  * Possible scenarios:
2558                  * 1. Stale disk was connected.
2559                  */
2560                 /* Previous state should be NEW. */
2561                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2562                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2563                     g_mirror_disk_state2str(disk->d_state)));
2564                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2565                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2566                     g_mirror_device_state2str(sc->sc_state),
2567                     g_mirror_get_diskname(disk),
2568                     g_mirror_disk_state2str(disk->d_state)));
2569                 /*
2570                  * STALE state is only possible if device is marked
2571                  * NOAUTOSYNC.
2572                  */
2573                 KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_NOAUTOSYNC) != 0,
2574                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2575                     g_mirror_device_state2str(sc->sc_state),
2576                     g_mirror_get_diskname(disk),
2577                     g_mirror_disk_state2str(disk->d_state)));
2578                 DISK_STATE_CHANGED();
2579
2580                 disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2581                 disk->d_state = state;
2582                 g_mirror_update_metadata(disk);
2583                 G_MIRROR_DEBUG(0, "Device %s: provider %s is stale.",
2584                     sc->sc_name, g_mirror_get_diskname(disk));
2585                 break;
2586         case G_MIRROR_DISK_STATE_SYNCHRONIZING:
2587                 /*
2588                  * Possible scenarios:
2589                  * 1. Disk which needs synchronization was connected.
2590                  */
2591                 /* Previous state should be NEW. */
2592                 KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2593                     ("Wrong disk state (%s, %s).", g_mirror_get_diskname(disk),
2594                     g_mirror_disk_state2str(disk->d_state)));
2595                 KASSERT(sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING,
2596                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2597                     g_mirror_device_state2str(sc->sc_state),
2598                     g_mirror_get_diskname(disk),
2599                     g_mirror_disk_state2str(disk->d_state)));
2600                 DISK_STATE_CHANGED();
2601
2602                 if (disk->d_state == G_MIRROR_DISK_STATE_NEW)
2603                         disk->d_flags &= ~G_MIRROR_DISK_FLAG_DIRTY;
2604                 disk->d_state = state;
2605                 if (sc->sc_provider != NULL) {
2606                         g_mirror_sync_start(disk);
2607                         g_mirror_update_metadata(disk);
2608                 }
2609                 break;
2610         case G_MIRROR_DISK_STATE_DISCONNECTED:
2611                 /*
2612                  * Possible scenarios:
2613                  * 1. Device wasn't running yet, but disk disappear.
2614                  * 2. Disk was active and disapppear.
2615                  * 3. Disk disappear during synchronization process.
2616                  */
2617                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING) {
2618                         /*
2619                          * Previous state should be ACTIVE, STALE or
2620                          * SYNCHRONIZING.
2621                          */
2622                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_ACTIVE ||
2623                             disk->d_state == G_MIRROR_DISK_STATE_STALE ||
2624                             disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING,
2625                             ("Wrong disk state (%s, %s).",
2626                             g_mirror_get_diskname(disk),
2627                             g_mirror_disk_state2str(disk->d_state)));
2628                 } else if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING) {
2629                         /* Previous state should be NEW. */
2630                         KASSERT(disk->d_state == G_MIRROR_DISK_STATE_NEW,
2631                             ("Wrong disk state (%s, %s).",
2632                             g_mirror_get_diskname(disk),
2633                             g_mirror_disk_state2str(disk->d_state)));
2634                         /*
2635                          * Reset bumping syncid if disk disappeared in STARTING
2636                          * state.
2637                          */
2638                         if ((sc->sc_bump_id & G_MIRROR_BUMP_SYNCID) != 0)
2639                                 sc->sc_bump_id &= ~G_MIRROR_BUMP_SYNCID;
2640 #ifdef  INVARIANTS
2641                 } else {
2642                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2643                             sc->sc_name,
2644                             g_mirror_device_state2str(sc->sc_state),
2645                             g_mirror_get_diskname(disk),
2646                             g_mirror_disk_state2str(disk->d_state)));
2647 #endif
2648                 }
2649                 DISK_STATE_CHANGED();
2650                 G_MIRROR_DEBUG(0, "Device %s: provider %s disconnected.",
2651                     sc->sc_name, g_mirror_get_diskname(disk));
2652
2653                 g_mirror_destroy_disk(disk);
2654                 break;
2655         case G_MIRROR_DISK_STATE_DESTROY:
2656             {
2657                 int error;
2658
2659                 error = g_mirror_clear_metadata(disk);
2660                 if (error != 0)
2661                         return (error);
2662                 DISK_STATE_CHANGED();
2663                 G_MIRROR_DEBUG(0, "Device %s: provider %s destroyed.",
2664                     sc->sc_name, g_mirror_get_diskname(disk));
2665
2666                 g_mirror_destroy_disk(disk);
2667                 sc->sc_ndisks--;
2668                 LIST_FOREACH(disk, &sc->sc_disks, d_next) {
2669                         g_mirror_update_metadata(disk);
2670                 }
2671                 break;
2672             }
2673         default:
2674                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2675                 break;
2676         }
2677         return (0);
2678 }
2679 #undef  DISK_STATE_CHANGED
2680
2681 int
2682 g_mirror_read_metadata(struct g_consumer *cp, struct g_mirror_metadata *md)
2683 {
2684         struct g_provider *pp;
2685         u_char *buf;
2686         int error;
2687
2688         g_topology_assert();
2689
2690         error = g_access(cp, 1, 0, 0);
2691         if (error != 0)
2692                 return (error);
2693         pp = cp->provider;
2694         g_topology_unlock();
2695         /* Metadata are stored on last sector. */
2696         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2697             &error);
2698         g_topology_lock();
2699         g_access(cp, -1, 0, 0);
2700         if (buf == NULL) {
2701                 G_MIRROR_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2702                     cp->provider->name, error);
2703                 return (error);
2704         }
2705
2706         /* Decode metadata. */
2707         error = mirror_metadata_decode(buf, md);
2708         g_free(buf);
2709         if (strcmp(md->md_magic, G_MIRROR_MAGIC) != 0)
2710                 return (EINVAL);
2711         if (md->md_version > G_MIRROR_VERSION) {
2712                 G_MIRROR_DEBUG(0,
2713                     "Kernel module is too old to handle metadata from %s.",
2714                     cp->provider->name);
2715                 return (EINVAL);
2716         }
2717         if (error != 0) {
2718                 G_MIRROR_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2719                     cp->provider->name);
2720                 return (error);
2721         }
2722
2723         return (0);
2724 }
2725
2726 static int
2727 g_mirror_check_metadata(struct g_mirror_softc *sc, struct g_provider *pp,
2728     struct g_mirror_metadata *md)
2729 {
2730
2731         if (g_mirror_id2disk(sc, md->md_did) != NULL) {
2732                 G_MIRROR_DEBUG(1, "Disk %s (id=%u) already exists, skipping.",
2733                     pp->name, md->md_did);
2734                 return (EEXIST);
2735         }
2736         if (md->md_all != sc->sc_ndisks) {
2737                 G_MIRROR_DEBUG(1,
2738                     "Invalid '%s' field on disk %s (device %s), skipping.",
2739                     "md_all", pp->name, sc->sc_name);
2740                 return (EINVAL);
2741         }
2742         if (md->md_slice != sc->sc_slice) {
2743                 G_MIRROR_DEBUG(1,
2744                     "Invalid '%s' field on disk %s (device %s), skipping.",
2745                     "md_slice", pp->name, sc->sc_name);
2746                 return (EINVAL);
2747         }
2748         if (md->md_balance != sc->sc_balance) {
2749                 G_MIRROR_DEBUG(1,
2750                     "Invalid '%s' field on disk %s (device %s), skipping.",
2751                     "md_balance", pp->name, sc->sc_name);
2752                 return (EINVAL);
2753         }
2754 #if 0
2755         if (md->md_mediasize != sc->sc_mediasize) {
2756                 G_MIRROR_DEBUG(1,
2757                     "Invalid '%s' field on disk %s (device %s), skipping.",
2758                     "md_mediasize", pp->name, sc->sc_name);
2759                 return (EINVAL);
2760         }
2761 #endif
2762         if (sc->sc_mediasize > pp->mediasize) {
2763                 G_MIRROR_DEBUG(1,
2764                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2765                     sc->sc_name);
2766                 return (EINVAL);
2767         }
2768         if (md->md_sectorsize != sc->sc_sectorsize) {
2769                 G_MIRROR_DEBUG(1,
2770                     "Invalid '%s' field on disk %s (device %s), skipping.",
2771                     "md_sectorsize", pp->name, sc->sc_name);
2772                 return (EINVAL);
2773         }
2774         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2775                 G_MIRROR_DEBUG(1,
2776                     "Invalid sector size of disk %s (device %s), skipping.",
2777                     pp->name, sc->sc_name);
2778                 return (EINVAL);
2779         }
2780         if ((md->md_mflags & ~G_MIRROR_DEVICE_FLAG_MASK) != 0) {
2781                 G_MIRROR_DEBUG(1,
2782                     "Invalid device flags on disk %s (device %s), skipping.",
2783                     pp->name, sc->sc_name);
2784                 return (EINVAL);
2785         }
2786         if ((md->md_dflags & ~G_MIRROR_DISK_FLAG_MASK) != 0) {
2787                 G_MIRROR_DEBUG(1,
2788                     "Invalid disk flags on disk %s (device %s), skipping.",
2789                     pp->name, sc->sc_name);
2790                 return (EINVAL);
2791         }
2792         return (0);
2793 }
2794
2795 int
2796 g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp,
2797     struct g_mirror_metadata *md)
2798 {
2799         struct g_mirror_disk *disk;
2800         int error;
2801
2802         g_topology_assert_not();
2803         G_MIRROR_DEBUG(2, "Adding disk %s.", pp->name);
2804
2805         error = g_mirror_check_metadata(sc, pp, md);
2806         if (error != 0)
2807                 return (error);
2808         if (sc->sc_state == G_MIRROR_DEVICE_STATE_RUNNING &&
2809             md->md_genid < sc->sc_genid) {
2810                 G_MIRROR_DEBUG(0, "Component %s (device %s) broken, skipping.",
2811                     pp->name, sc->sc_name);
2812                 return (EINVAL);
2813         }
2814         disk = g_mirror_init_disk(sc, pp, md, &error);
2815         if (disk == NULL)
2816                 return (error);
2817         error = g_mirror_event_send(disk, G_MIRROR_DISK_STATE_NEW,
2818             G_MIRROR_EVENT_WAIT);
2819         if (error != 0)
2820                 return (error);
2821         if (md->md_version < G_MIRROR_VERSION) {
2822                 G_MIRROR_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
2823                     pp->name, md->md_version, G_MIRROR_VERSION);
2824                 g_mirror_update_metadata(disk);
2825         }
2826         return (0);
2827 }
2828
2829 static void
2830 g_mirror_destroy_delayed(void *arg, int flag)
2831 {
2832         struct g_mirror_softc *sc;
2833         int error;
2834
2835         if (flag == EV_CANCEL) {
2836                 G_MIRROR_DEBUG(1, "Destroying canceled.");
2837                 return;
2838         }
2839         sc = arg;
2840         g_topology_unlock();
2841         sx_xlock(&sc->sc_lock);
2842         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0,
2843             ("DESTROY flag set on %s.", sc->sc_name));
2844         KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0,
2845             ("DESTROYING flag not set on %s.", sc->sc_name));
2846         G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name);
2847         error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT);
2848         if (error != 0) {
2849                 G_MIRROR_DEBUG(0, "Cannot destroy %s (error=%d).",
2850                     sc->sc_name, error);
2851                 sx_xunlock(&sc->sc_lock);
2852         }
2853         g_topology_lock();
2854 }
2855
2856 static int
2857 g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
2858 {
2859         struct g_mirror_softc *sc;
2860         int dcr, dcw, dce, error = 0;
2861
2862         g_topology_assert();
2863         G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
2864             acw, ace);
2865
2866         sc = pp->geom->softc;
2867         if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
2868                 return (0);
2869         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
2870
2871         dcr = pp->acr + acr;
2872         dcw = pp->acw + acw;
2873         dce = pp->ace + ace;
2874
2875         g_topology_unlock();
2876         sx_xlock(&sc->sc_lock);
2877         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 ||
2878             LIST_EMPTY(&sc->sc_disks)) {
2879                 if (acr > 0 || acw > 0 || ace > 0)
2880                         error = ENXIO;
2881                 goto end;
2882         }
2883         if (dcw == 0)
2884                 g_mirror_idle(sc, dcw);
2885         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
2886                 if (acr > 0 || acw > 0 || ace > 0) {
2887                         error = ENXIO;
2888                         goto end;
2889                 }
2890                 if (dcr == 0 && dcw == 0 && dce == 0) {
2891                         g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK,
2892                             sc, NULL);
2893                 }
2894         }
2895 end:
2896         sx_xunlock(&sc->sc_lock);
2897         g_topology_lock();
2898         return (error);
2899 }
2900
2901 static struct g_geom *
2902 g_mirror_create(struct g_class *mp, const struct g_mirror_metadata *md)
2903 {
2904         struct g_mirror_softc *sc;
2905         struct g_geom *gp;
2906         int error, timeout;
2907
2908         g_topology_assert();
2909         G_MIRROR_DEBUG(1, "Creating device %s (id=%u).", md->md_name,
2910             md->md_mid);
2911
2912         /* One disk is minimum. */
2913         if (md->md_all < 1)
2914                 return (NULL);
2915         /*
2916          * Action geom.
2917          */
2918         gp = g_new_geomf(mp, "%s", md->md_name);
2919         sc = malloc(sizeof(*sc), M_MIRROR, M_WAITOK | M_ZERO);
2920         gp->start = g_mirror_start;
2921         gp->orphan = g_mirror_orphan;
2922         gp->access = g_mirror_access;
2923         gp->dumpconf = g_mirror_dumpconf;
2924
2925         sc->sc_id = md->md_mid;
2926         sc->sc_slice = md->md_slice;
2927         sc->sc_balance = md->md_balance;
2928         sc->sc_mediasize = md->md_mediasize;
2929         sc->sc_sectorsize = md->md_sectorsize;
2930         sc->sc_ndisks = md->md_all;
2931         sc->sc_flags = md->md_mflags;
2932         sc->sc_bump_id = 0;
2933         sc->sc_idle = 1;
2934         sc->sc_last_write = time_uptime;
2935         sc->sc_writes = 0;
2936         sx_init(&sc->sc_lock, "gmirror:lock");
2937         bioq_init(&sc->sc_queue);
2938         mtx_init(&sc->sc_queue_mtx, "gmirror:queue", NULL, MTX_DEF);
2939         bioq_init(&sc->sc_regular_delayed);
2940         bioq_init(&sc->sc_inflight);
2941         bioq_init(&sc->sc_sync_delayed);
2942         LIST_INIT(&sc->sc_disks);
2943         TAILQ_INIT(&sc->sc_events);
2944         mtx_init(&sc->sc_events_mtx, "gmirror:events", NULL, MTX_DEF);
2945         callout_init(&sc->sc_callout, CALLOUT_MPSAFE);
2946         mtx_init(&sc->sc_done_mtx, "gmirror:done", NULL, MTX_DEF);
2947         sc->sc_state = G_MIRROR_DEVICE_STATE_STARTING;
2948         gp->softc = sc;
2949         sc->sc_geom = gp;
2950         sc->sc_provider = NULL;
2951         /*
2952          * Synchronization geom.
2953          */
2954         gp = g_new_geomf(mp, "%s.sync", md->md_name);
2955         gp->softc = sc;
2956         gp->orphan = g_mirror_orphan;
2957         sc->sc_sync.ds_geom = gp;
2958         sc->sc_sync.ds_ndisks = 0;
2959         error = kproc_create(g_mirror_worker, sc, &sc->sc_worker, 0, 0,
2960             "g_mirror %s", md->md_name);
2961         if (error != 0) {
2962                 G_MIRROR_DEBUG(1, "Cannot create kernel thread for %s.",
2963                     sc->sc_name);
2964                 g_destroy_geom(sc->sc_sync.ds_geom);
2965                 mtx_destroy(&sc->sc_done_mtx);
2966                 mtx_destroy(&sc->sc_events_mtx);
2967                 mtx_destroy(&sc->sc_queue_mtx);
2968                 sx_destroy(&sc->sc_lock);
2969                 g_destroy_geom(sc->sc_geom);
2970                 free(sc, M_MIRROR);
2971                 return (NULL);
2972         }
2973
2974         G_MIRROR_DEBUG(1, "Device %s created (%u components, id=%u).",
2975             sc->sc_name, sc->sc_ndisks, sc->sc_id);
2976
2977         sc->sc_rootmount = root_mount_hold("GMIRROR");
2978         G_MIRROR_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
2979         /*
2980          * Run timeout.
2981          */
2982         timeout = g_mirror_timeout * hz;
2983         callout_reset(&sc->sc_callout, timeout, g_mirror_go, sc);
2984         return (sc->sc_geom);
2985 }
2986
2987 int
2988 g_mirror_destroy(struct g_mirror_softc *sc, int how)
2989 {
2990         struct g_mirror_disk *disk;
2991         struct g_provider *pp;
2992
2993         g_topology_assert_not();
2994         if (sc == NULL)
2995                 return (ENXIO);
2996         sx_assert(&sc->sc_lock, SX_XLOCKED);
2997
2998         pp = sc->sc_provider;
2999         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
3000                 switch (how) {
3001                 case G_MIRROR_DESTROY_SOFT:
3002                         G_MIRROR_DEBUG(1,
3003                             "Device %s is still open (r%dw%de%d).", pp->name,
3004                             pp->acr, pp->acw, pp->ace);
3005                         return (EBUSY);
3006                 case G_MIRROR_DESTROY_DELAYED:
3007                         G_MIRROR_DEBUG(1,
3008                             "Device %s will be destroyed on last close.",
3009                             pp->name);
3010                         LIST_FOREACH(disk, &sc->sc_disks, d_next) {
3011                                 if (disk->d_state ==
3012                                     G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3013                                         g_mirror_sync_stop(disk, 1);
3014                                 }
3015                         }
3016                         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING;
3017                         return (EBUSY);
3018                 case G_MIRROR_DESTROY_HARD:
3019                         G_MIRROR_DEBUG(1, "Device %s is still open, so it "
3020                             "can't be definitely removed.", pp->name);
3021                 }
3022         }
3023
3024         g_topology_lock();
3025         if (sc->sc_geom->softc == NULL) {
3026                 g_topology_unlock();
3027                 return (0);
3028         }
3029         sc->sc_geom->softc = NULL;
3030         sc->sc_sync.ds_geom->softc = NULL;
3031         g_topology_unlock();
3032
3033         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROY;
3034         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_WAIT;
3035         G_MIRROR_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3036         sx_xunlock(&sc->sc_lock);
3037         mtx_lock(&sc->sc_queue_mtx);
3038         wakeup(sc);
3039         mtx_unlock(&sc->sc_queue_mtx);
3040         G_MIRROR_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3041         while (sc->sc_worker != NULL)
3042                 tsleep(&sc->sc_worker, PRIBIO, "m:destroy", hz / 5);
3043         G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3044         sx_xlock(&sc->sc_lock);
3045         g_mirror_destroy_device(sc);
3046         free(sc, M_MIRROR);
3047         return (0);
3048 }
3049
3050 static void
3051 g_mirror_taste_orphan(struct g_consumer *cp)
3052 {
3053
3054         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3055             cp->provider->name));
3056 }
3057
3058 static struct g_geom *
3059 g_mirror_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3060 {
3061         struct g_mirror_metadata md;
3062         struct g_mirror_softc *sc;
3063         struct g_consumer *cp;
3064         struct g_geom *gp;
3065         int error;
3066
3067         g_topology_assert();
3068         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3069         G_MIRROR_DEBUG(2, "Tasting %s.", pp->name);
3070
3071         gp = g_new_geomf(mp, "mirror:taste");
3072         /*
3073          * This orphan function should be never called.
3074          */
3075         gp->orphan = g_mirror_taste_orphan;
3076         cp = g_new_consumer(gp);
3077         g_attach(cp, pp);
3078         error = g_mirror_read_metadata(cp, &md);
3079         g_detach(cp);
3080         g_destroy_consumer(cp);
3081         g_destroy_geom(gp);
3082         if (error != 0)
3083                 return (NULL);
3084         gp = NULL;
3085
3086         if (md.md_provider[0] != '\0' &&
3087             !g_compare_names(md.md_provider, pp->name))
3088                 return (NULL);
3089         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3090                 return (NULL);
3091         if ((md.md_dflags & G_MIRROR_DISK_FLAG_INACTIVE) != 0) {
3092                 G_MIRROR_DEBUG(0,
3093                     "Device %s: provider %s marked as inactive, skipping.",
3094                     md.md_name, pp->name);
3095                 return (NULL);
3096         }
3097         if (g_mirror_debug >= 2)
3098                 mirror_metadata_dump(&md);
3099
3100         /*
3101          * Let's check if device already exists.
3102          */
3103         sc = NULL;
3104         LIST_FOREACH(gp, &mp->geom, geom) {
3105                 sc = gp->softc;
3106                 if (sc == NULL)
3107                         continue;
3108                 if (sc->sc_sync.ds_geom == gp)
3109                         continue;
3110                 if (strcmp(md.md_name, sc->sc_name) != 0)
3111                         continue;
3112                 if (md.md_mid != sc->sc_id) {
3113                         G_MIRROR_DEBUG(0, "Device %s already configured.",
3114                             sc->sc_name);
3115                         return (NULL);
3116                 }
3117                 break;
3118         }
3119         if (gp == NULL) {
3120                 gp = g_mirror_create(mp, &md);
3121                 if (gp == NULL) {
3122                         G_MIRROR_DEBUG(0, "Cannot create device %s.",
3123                             md.md_name);
3124                         return (NULL);
3125                 }
3126                 sc = gp->softc;
3127         }
3128         G_MIRROR_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3129         g_topology_unlock();
3130         sx_xlock(&sc->sc_lock);
3131         sc->sc_flags |= G_MIRROR_DEVICE_FLAG_TASTING;
3132         error = g_mirror_add_disk(sc, pp, &md);
3133         if (error != 0) {
3134                 G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3135                     pp->name, gp->name, error);
3136                 if (LIST_EMPTY(&sc->sc_disks)) {
3137                         g_cancel_event(sc);
3138                         g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3139                         g_topology_lock();
3140                         return (NULL);
3141                 }
3142                 gp = NULL;
3143         }
3144         sc->sc_flags &= ~G_MIRROR_DEVICE_FLAG_TASTING;
3145         if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) {
3146                 g_mirror_destroy(sc, G_MIRROR_DESTROY_HARD);
3147                 g_topology_lock();
3148                 return (NULL);
3149         }
3150         sx_xunlock(&sc->sc_lock);
3151         g_topology_lock();
3152         return (gp);
3153 }
3154
3155 static void
3156 g_mirror_resize(struct g_consumer *cp)
3157 {
3158         struct g_mirror_disk *disk;
3159
3160         g_topology_assert();
3161         g_trace(G_T_TOPOLOGY, "%s(%s)", __func__, cp->provider->name);
3162
3163         disk = cp->private;
3164         if (disk == NULL)
3165                 return;
3166         g_topology_unlock();
3167         g_mirror_update_metadata(disk);
3168         g_topology_lock();
3169 }
3170
3171 static int
3172 g_mirror_destroy_geom(struct gctl_req *req __unused,
3173     struct g_class *mp __unused, struct g_geom *gp)
3174 {
3175         struct g_mirror_softc *sc;
3176         int error;
3177
3178         g_topology_unlock();
3179         sc = gp->softc;
3180         sx_xlock(&sc->sc_lock);
3181         g_cancel_event(sc);
3182         error = g_mirror_destroy(gp->softc, G_MIRROR_DESTROY_SOFT);
3183         if (error != 0)
3184                 sx_xunlock(&sc->sc_lock);
3185         g_topology_lock();
3186         return (error);
3187 }
3188
3189 static void
3190 g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3191     struct g_consumer *cp, struct g_provider *pp)
3192 {
3193         struct g_mirror_softc *sc;
3194
3195         g_topology_assert();
3196
3197         sc = gp->softc;
3198         if (sc == NULL)
3199                 return;
3200         /* Skip synchronization geom. */
3201         if (gp == sc->sc_sync.ds_geom)
3202                 return;
3203         if (pp != NULL) {
3204                 /* Nothing here. */
3205         } else if (cp != NULL) {
3206                 struct g_mirror_disk *disk;
3207
3208                 disk = cp->private;
3209                 if (disk == NULL)
3210                         return;
3211                 g_topology_unlock();
3212                 sx_xlock(&sc->sc_lock);
3213                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)disk->d_id);
3214                 if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) {
3215                         sbuf_printf(sb, "%s<Synchronized>", indent);
3216                         if (disk->d_sync.ds_offset == 0)
3217                                 sbuf_printf(sb, "0%%");
3218                         else {
3219                                 sbuf_printf(sb, "%u%%",
3220                                     (u_int)((disk->d_sync.ds_offset * 100) /
3221                                     sc->sc_provider->mediasize));
3222                         }
3223                         sbuf_printf(sb, "</Synchronized>\n");
3224                         if (disk->d_sync.ds_offset > 0) {
3225                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3226                                     "</BytesSynced>\n", indent,
3227                                     (intmax_t)disk->d_sync.ds_offset);
3228                         }
3229                 }
3230                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3231                     disk->d_sync.ds_syncid);
3232                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent,
3233                     disk->d_genid);
3234                 sbuf_printf(sb, "%s<Flags>", indent);
3235                 if (disk->d_flags == 0)
3236                         sbuf_printf(sb, "NONE");
3237                 else {
3238                         int first = 1;
3239
3240 #define ADD_FLAG(flag, name)    do {                                    \
3241         if ((disk->d_flags & (flag)) != 0) {                            \
3242                 if (!first)                                             \
3243                         sbuf_printf(sb, ", ");                          \
3244                 else                                                    \
3245                         first = 0;                                      \
3246                 sbuf_printf(sb, name);                                  \
3247         }                                                               \
3248 } while (0)
3249                         ADD_FLAG(G_MIRROR_DISK_FLAG_DIRTY, "DIRTY");
3250                         ADD_FLAG(G_MIRROR_DISK_FLAG_HARDCODED, "HARDCODED");
3251                         ADD_FLAG(G_MIRROR_DISK_FLAG_INACTIVE, "INACTIVE");
3252                         ADD_FLAG(G_MIRROR_DISK_FLAG_SYNCHRONIZING,
3253                             "SYNCHRONIZING");
3254                         ADD_FLAG(G_MIRROR_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3255                         ADD_FLAG(G_MIRROR_DISK_FLAG_BROKEN, "BROKEN");
3256 #undef  ADD_FLAG
3257                 }
3258                 sbuf_printf(sb, "</Flags>\n");
3259                 sbuf_printf(sb, "%s<Priority>%u</Priority>\n", indent,
3260                     disk->d_priority);
3261                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3262                     g_mirror_disk_state2str(disk->d_state));
3263                 sx_xunlock(&sc->sc_lock);
3264                 g_topology_lock();
3265         } else {
3266                 g_topology_unlock();
3267                 sx_xlock(&sc->sc_lock);
3268                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3269                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3270                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3271                 sbuf_printf(sb, "%s<Flags>", indent);
3272                 if (sc->sc_flags == 0)
3273                         sbuf_printf(sb, "NONE");
3274                 else {
3275                         int first = 1;
3276
3277 #define ADD_FLAG(flag, name)    do {                                    \
3278         if ((sc->sc_flags & (flag)) != 0) {                             \
3279                 if (!first)                                             \
3280                         sbuf_printf(sb, ", ");                          \
3281                 else                                                    \
3282                         first = 0;                                      \
3283                 sbuf_printf(sb, name);                                  \
3284         }                                                               \
3285 } while (0)
3286                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3287                         ADD_FLAG(G_MIRROR_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3288 #undef  ADD_FLAG
3289                 }
3290                 sbuf_printf(sb, "</Flags>\n");
3291                 sbuf_printf(sb, "%s<Slice>%u</Slice>\n", indent,
3292                     (u_int)sc->sc_slice);
3293                 sbuf_printf(sb, "%s<Balance>%s</Balance>\n", indent,
3294                     balance_name(sc->sc_balance));
3295                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3296                     sc->sc_ndisks);
3297                 sbuf_printf(sb, "%s<State>", indent);
3298                 if (sc->sc_state == G_MIRROR_DEVICE_STATE_STARTING)
3299                         sbuf_printf(sb, "%s", "STARTING");
3300                 else if (sc->sc_ndisks ==
3301                     g_mirror_ndisks(sc, G_MIRROR_DISK_STATE_ACTIVE))
3302                         sbuf_printf(sb, "%s", "COMPLETE");
3303                 else
3304                         sbuf_printf(sb, "%s", "DEGRADED");
3305                 sbuf_printf(sb, "</State>\n");
3306                 sx_xunlock(&sc->sc_lock);
3307                 g_topology_lock();
3308         }
3309 }
3310
3311 static void
3312 g_mirror_shutdown_post_sync(void *arg, int howto)
3313 {
3314         struct g_class *mp;
3315         struct g_geom *gp, *gp2;
3316         struct g_mirror_softc *sc;
3317         int error;
3318
3319         mp = arg;
3320         DROP_GIANT();
3321         g_topology_lock();
3322         g_mirror_shutdown = 1;
3323         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3324                 if ((sc = gp->softc) == NULL)
3325                         continue;
3326                 /* Skip synchronization geom. */
3327                 if (gp == sc->sc_sync.ds_geom)
3328                         continue;
3329                 g_topology_unlock();
3330                 sx_xlock(&sc->sc_lock);
3331                 g_mirror_idle(sc, -1);
3332                 g_cancel_event(sc);
3333                 error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
3334                 if (error != 0)
3335                         sx_xunlock(&sc->sc_lock);
3336                 g_topology_lock();
3337         }
3338         g_topology_unlock();
3339         PICKUP_GIANT();
3340 }
3341
3342 static void
3343 g_mirror_init(struct g_class *mp)
3344 {
3345
3346         g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3347             g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3348         if (g_mirror_post_sync == NULL)
3349                 G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
3350 }
3351
3352 static void
3353 g_mirror_fini(struct g_class *mp)
3354 {
3355
3356         if (g_mirror_post_sync != NULL)
3357                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
3358 }
3359
3360 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);