sys/geom/raid3/g_raid3.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2004-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/module.h>
  36 #include <sys/limits.h>
  37 #include <sys/lock.h>
  38 #include <sys/mutex.h>
  39 #include <sys/bio.h>
  40 #include <sys/sbuf.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/malloc.h>
  43 #include <sys/eventhandler.h>
  44 #include <vm/uma.h>
  45 #include <geom/geom.h>
  46 #include <sys/proc.h>
  47 #include <sys/kthread.h>
  48 #include <sys/sched.h>
  49 #include <geom/raid3/g_raid3.h>
  50
  51 FEATURE(geom_raid3, "GEOM RAID-3 functionality");
  52
  53 static MALLOC_DEFINE(M_RAID3, "raid3_data", "GEOM_RAID3 Data");
  54
  55 SYSCTL_DECL(_kern_geom);
  56 static SYSCTL_NODE(_kern_geom, OID_AUTO, raid3, CTLFLAG_RW, 0,
  57     "GEOM_RAID3 stuff");
  58 u_int g_raid3_debug = 0;
  59 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, debug, CTLFLAG_RWTUN, &g_raid3_debug, 0,
  60     "Debug level");
  61 static u_int g_raid3_timeout = 4;
  62 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, timeout, CTLFLAG_RWTUN, &g_raid3_timeout,
  63     0, "Time to wait on all raid3 components");
  64 static u_int g_raid3_idletime = 5;
  65 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, idletime, CTLFLAG_RWTUN,
  66     &g_raid3_idletime, 0, "Mark components as clean when idling");
  67 static u_int g_raid3_disconnect_on_failure = 1;
  68 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, disconnect_on_failure, CTLFLAG_RWTUN,
  69     &g_raid3_disconnect_on_failure, 0, "Disconnect component on I/O failure.");
  70 static u_int g_raid3_syncreqs = 2;
  71 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
  72     &g_raid3_syncreqs, 0, "Parallel synchronization I/O requests.");
  73 static u_int g_raid3_use_malloc = 0;
  74 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, use_malloc, CTLFLAG_RDTUN,
  75     &g_raid3_use_malloc, 0, "Use malloc(9) instead of uma(9).");
  76
  77 static u_int g_raid3_n64k = 50;
  78 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n64k, CTLFLAG_RDTUN, &g_raid3_n64k, 0,
  79     "Maximum number of 64kB allocations");
  80 static u_int g_raid3_n16k = 200;
  81 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n16k, CTLFLAG_RDTUN, &g_raid3_n16k, 0,
  82     "Maximum number of 16kB allocations");
  83 static u_int g_raid3_n4k = 1200;
  84 SYSCTL_UINT(_kern_geom_raid3, OID_AUTO, n4k, CTLFLAG_RDTUN, &g_raid3_n4k, 0,
  85     "Maximum number of 4kB allocations");
  86
  87 static SYSCTL_NODE(_kern_geom_raid3, OID_AUTO, stat, CTLFLAG_RW, 0,
  88     "GEOM_RAID3 statistics");
  89 static u_int g_raid3_parity_mismatch = 0;
  90 SYSCTL_UINT(_kern_geom_raid3_stat, OID_AUTO, parity_mismatch, CTLFLAG_RD,
  91     &g_raid3_parity_mismatch, 0, "Number of failures in VERIFY mode");
  92
  93 #define MSLEEP(ident, mtx, priority, wmesg, timeout)    do {            \
  94         G_RAID3_DEBUG(4, "%s: Sleeping %p.", __func__, (ident));        \
  95         msleep((ident), (mtx), (priority), (wmesg), (timeout));         \
  96         G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, (ident));        \
  97 } while (0)
  98
  99 static eventhandler_tag g_raid3_post_sync = NULL;
 100 static int g_raid3_shutdown = 0;
 101
 102 static int g_raid3_destroy_geom(struct gctl_req *req, struct g_class *mp,
 103     struct g_geom *gp);
 104 static g_taste_t g_raid3_taste;
 105 static void g_raid3_init(struct g_class *mp);
 106 static void g_raid3_fini(struct g_class *mp);
 107
 108 struct g_class g_raid3_class = {
 109         .name = G_RAID3_CLASS_NAME,
 110         .version = G_VERSION,
 111         .ctlreq = g_raid3_config,
 112         .taste = g_raid3_taste,
 113         .destroy_geom = g_raid3_destroy_geom,
 114         .init = g_raid3_init,
 115         .fini = g_raid3_fini
 116 };
 117
 118
 119 static void g_raid3_destroy_provider(struct g_raid3_softc *sc);
 120 static int g_raid3_update_disk(struct g_raid3_disk *disk, u_int state);
 121 static void g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force);
 122 static void g_raid3_dumpconf(struct sbuf *sb, const char *indent,
 123     struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
 124 static void g_raid3_sync_stop(struct g_raid3_softc *sc, int type);
 125 static int g_raid3_register_request(struct bio *pbp);
 126 static void g_raid3_sync_release(struct g_raid3_softc *sc);
 127
 128
 129 static const char *
 130 g_raid3_disk_state2str(int state)
 131 {
 132
 133         switch (state) {
 134         case G_RAID3_DISK_STATE_NODISK:
 135                 return ("NODISK");
 136         case G_RAID3_DISK_STATE_NONE:
 137                 return ("NONE");
 138         case G_RAID3_DISK_STATE_NEW:
 139                 return ("NEW");
 140         case G_RAID3_DISK_STATE_ACTIVE:
 141                 return ("ACTIVE");
 142         case G_RAID3_DISK_STATE_STALE:
 143                 return ("STALE");
 144         case G_RAID3_DISK_STATE_SYNCHRONIZING:
 145                 return ("SYNCHRONIZING");
 146         case G_RAID3_DISK_STATE_DISCONNECTED:
 147                 return ("DISCONNECTED");
 148         default:
 149                 return ("INVALID");
 150         }
 151 }
 152
 153 static const char *
 154 g_raid3_device_state2str(int state)
 155 {
 156
 157         switch (state) {
 158         case G_RAID3_DEVICE_STATE_STARTING:
 159                 return ("STARTING");
 160         case G_RAID3_DEVICE_STATE_DEGRADED:
 161                 return ("DEGRADED");
 162         case G_RAID3_DEVICE_STATE_COMPLETE:
 163                 return ("COMPLETE");
 164         default:
 165                 return ("INVALID");
 166         }
 167 }
 168
 169 const char *
 170 g_raid3_get_diskname(struct g_raid3_disk *disk)
 171 {
 172
 173         if (disk->d_consumer == NULL || disk->d_consumer->provider == NULL)
 174                 return ("[unknown]");
 175         return (disk->d_name);
 176 }
 177
 178 static void *
 179 g_raid3_alloc(struct g_raid3_softc *sc, size_t size, int flags)
 180 {
 181         void *ptr;
 182         enum g_raid3_zones zone;
 183
 184         if (g_raid3_use_malloc ||
 185             (zone = g_raid3_zone(size)) == G_RAID3_NUM_ZONES)
 186                 ptr = malloc(size, M_RAID3, flags);
 187         else {
 188                 ptr = uma_zalloc_arg(sc->sc_zones[zone].sz_zone,
 189                    &sc->sc_zones[zone], flags);
 190                 sc->sc_zones[zone].sz_requested++;
 191                 if (ptr == NULL)
 192                         sc->sc_zones[zone].sz_failed++;
 193         }
 194         return (ptr);
 195 }
 196
 197 static void
 198 g_raid3_free(struct g_raid3_softc *sc, void *ptr, size_t size)
 199 {
 200         enum g_raid3_zones zone;
 201
 202         if (g_raid3_use_malloc ||
 203             (zone = g_raid3_zone(size)) == G_RAID3_NUM_ZONES)
 204                 free(ptr, M_RAID3);
 205         else {
 206                 uma_zfree_arg(sc->sc_zones[zone].sz_zone,
 207                     ptr, &sc->sc_zones[zone]);
 208         }
 209 }
 210
 211 static int
 212 g_raid3_uma_ctor(void *mem, int size, void *arg, int flags)
 213 {
 214         struct g_raid3_zone *sz = arg;
 215
 216         if (sz->sz_max > 0 && sz->sz_inuse == sz->sz_max)
 217                 return (ENOMEM);
 218         sz->sz_inuse++;
 219         return (0);
 220 }
 221
 222 static void
 223 g_raid3_uma_dtor(void *mem, int size, void *arg)
 224 {
 225         struct g_raid3_zone *sz = arg;
 226
 227         sz->sz_inuse--;
 228 }
 229
 230 #define g_raid3_xor(src, dst, size)                                     \
 231         _g_raid3_xor((uint64_t *)(src),                                 \
 232             (uint64_t *)(dst), (size_t)size)
 233 static void
 234 _g_raid3_xor(uint64_t *src, uint64_t *dst, size_t size)
 235 {
 236
 237         KASSERT((size % 128) == 0, ("Invalid size: %zu.", size));
 238         for (; size > 0; size -= 128) {
 239                 *dst++ ^= (*src++);
 240                 *dst++ ^= (*src++);
 241                 *dst++ ^= (*src++);
 242                 *dst++ ^= (*src++);
 243                 *dst++ ^= (*src++);
 244                 *dst++ ^= (*src++);
 245                 *dst++ ^= (*src++);
 246                 *dst++ ^= (*src++);
 247                 *dst++ ^= (*src++);
 248                 *dst++ ^= (*src++);
 249                 *dst++ ^= (*src++);
 250                 *dst++ ^= (*src++);
 251                 *dst++ ^= (*src++);
 252                 *dst++ ^= (*src++);
 253                 *dst++ ^= (*src++);
 254                 *dst++ ^= (*src++);
 255         }
 256 }
 257
 258 static int
 259 g_raid3_is_zero(struct bio *bp)
 260 {
 261         static const uint64_t zeros[] = {
 262             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 263         };
 264         u_char *addr;
 265         ssize_t size;
 266
 267         size = bp->bio_length;
 268         addr = (u_char *)bp->bio_data;
 269         for (; size > 0; size -= sizeof(zeros), addr += sizeof(zeros)) {
 270                 if (bcmp(addr, zeros, sizeof(zeros)) != 0)
 271                         return (0);
 272         }
 273         return (1);
 274 }
 275
 276 /*
 277  * --- Events handling functions ---
 278  * Events in geom_raid3 are used to maintain disks and device status
 279  * from one thread to simplify locking.
 280  */
 281 static void
 282 g_raid3_event_free(struct g_raid3_event *ep)
 283 {
 284
 285         free(ep, M_RAID3);
 286 }
 287
 288 int
 289 g_raid3_event_send(void *arg, int state, int flags)
 290 {
 291         struct g_raid3_softc *sc;
 292         struct g_raid3_disk *disk;
 293         struct g_raid3_event *ep;
 294         int error;
 295
 296         ep = malloc(sizeof(*ep), M_RAID3, M_WAITOK);
 297         G_RAID3_DEBUG(4, "%s: Sending event %p.", __func__, ep);
 298         if ((flags & G_RAID3_EVENT_DEVICE) != 0) {
 299                 disk = NULL;
 300                 sc = arg;
 301         } else {
 302                 disk = arg;
 303                 sc = disk->d_softc;
 304         }
 305         ep->e_disk = disk;
 306         ep->e_state = state;
 307         ep->e_flags = flags;
 308         ep->e_error = 0;
 309         mtx_lock(&sc->sc_events_mtx);
 310         TAILQ_INSERT_TAIL(&sc->sc_events, ep, e_next);
 311         mtx_unlock(&sc->sc_events_mtx);
 312         G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, sc);
 313         mtx_lock(&sc->sc_queue_mtx);
 314         wakeup(sc);
 315         wakeup(&sc->sc_queue);
 316         mtx_unlock(&sc->sc_queue_mtx);
 317         if ((flags & G_RAID3_EVENT_DONTWAIT) != 0)
 318                 return (0);
 319         sx_assert(&sc->sc_lock, SX_XLOCKED);
 320         G_RAID3_DEBUG(4, "%s: Sleeping %p.", __func__, ep);
 321         sx_xunlock(&sc->sc_lock);
 322         while ((ep->e_flags & G_RAID3_EVENT_DONE) == 0) {
 323                 mtx_lock(&sc->sc_events_mtx);
 324                 MSLEEP(ep, &sc->sc_events_mtx, PRIBIO | PDROP, "r3:event",
 325                     hz * 5);
 326         }
 327         error = ep->e_error;
 328         g_raid3_event_free(ep);
 329         sx_xlock(&sc->sc_lock);
 330         return (error);
 331 }
 332
 333 static struct g_raid3_event *
 334 g_raid3_event_get(struct g_raid3_softc *sc)
 335 {
 336         struct g_raid3_event *ep;
 337
 338         mtx_lock(&sc->sc_events_mtx);
 339         ep = TAILQ_FIRST(&sc->sc_events);
 340         mtx_unlock(&sc->sc_events_mtx);
 341         return (ep);
 342 }
 343
 344 static void
 345 g_raid3_event_remove(struct g_raid3_softc *sc, struct g_raid3_event *ep)
 346 {
 347
 348         mtx_lock(&sc->sc_events_mtx);
 349         TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 350         mtx_unlock(&sc->sc_events_mtx);
 351 }
 352
 353 static void
 354 g_raid3_event_cancel(struct g_raid3_disk *disk)
 355 {
 356         struct g_raid3_softc *sc;
 357         struct g_raid3_event *ep, *tmpep;
 358
 359         sc = disk->d_softc;
 360         sx_assert(&sc->sc_lock, SX_XLOCKED);
 361
 362         mtx_lock(&sc->sc_events_mtx);
 363         TAILQ_FOREACH_SAFE(ep, &sc->sc_events, e_next, tmpep) {
 364                 if ((ep->e_flags & G_RAID3_EVENT_DEVICE) != 0)
 365                         continue;
 366                 if (ep->e_disk != disk)
 367                         continue;
 368                 TAILQ_REMOVE(&sc->sc_events, ep, e_next);
 369                 if ((ep->e_flags & G_RAID3_EVENT_DONTWAIT) != 0)
 370                         g_raid3_event_free(ep);
 371                 else {
 372                         ep->e_error = ECANCELED;
 373                         wakeup(ep);
 374                 }
 375         }
 376         mtx_unlock(&sc->sc_events_mtx);
 377 }
 378
 379 /*
 380  * Return the number of disks in the given state.
 381  * If state is equal to -1, count all connected disks.
 382  */
 383 u_int
 384 g_raid3_ndisks(struct g_raid3_softc *sc, int state)
 385 {
 386         struct g_raid3_disk *disk;
 387         u_int n, ndisks;
 388
 389         sx_assert(&sc->sc_lock, SX_LOCKED);
 390
 391         for (n = ndisks = 0; n < sc->sc_ndisks; n++) {
 392                 disk = &sc->sc_disks[n];
 393                 if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
 394                         continue;
 395                 if (state == -1 || disk->d_state == state)
 396                         ndisks++;
 397         }
 398         return (ndisks);
 399 }
 400
 401 static u_int
 402 g_raid3_nrequests(struct g_raid3_softc *sc, struct g_consumer *cp)
 403 {
 404         struct bio *bp;
 405         u_int nreqs = 0;
 406
 407         mtx_lock(&sc->sc_queue_mtx);
 408         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 409                 if (bp->bio_from == cp)
 410                         nreqs++;
 411         }
 412         mtx_unlock(&sc->sc_queue_mtx);
 413         return (nreqs);
 414 }
 415
 416 static int
 417 g_raid3_is_busy(struct g_raid3_softc *sc, struct g_consumer *cp)
 418 {
 419
 420         if (cp->index > 0) {
 421                 G_RAID3_DEBUG(2,
 422                     "I/O requests for %s exist, can't destroy it now.",
 423                     cp->provider->name);
 424                 return (1);
 425         }
 426         if (g_raid3_nrequests(sc, cp) > 0) {
 427                 G_RAID3_DEBUG(2,
 428                     "I/O requests for %s in queue, can't destroy it now.",
 429                     cp->provider->name);
 430                 return (1);
 431         }
 432         return (0);
 433 }
 434
 435 static void
 436 g_raid3_destroy_consumer(void *arg, int flags __unused)
 437 {
 438         struct g_consumer *cp;
 439
 440         g_topology_assert();
 441
 442         cp = arg;
 443         G_RAID3_DEBUG(1, "Consumer %s destroyed.", cp->provider->name);
 444         g_detach(cp);
 445         g_destroy_consumer(cp);
 446 }
 447
 448 static void
 449 g_raid3_kill_consumer(struct g_raid3_softc *sc, struct g_consumer *cp)
 450 {
 451         struct g_provider *pp;
 452         int retaste_wait;
 453
 454         g_topology_assert();
 455
 456         cp->private = NULL;
 457         if (g_raid3_is_busy(sc, cp))
 458                 return;
 459         G_RAID3_DEBUG(2, "Consumer %s destroyed.", cp->provider->name);
 460         pp = cp->provider;
 461         retaste_wait = 0;
 462         if (cp->acw == 1) {
 463                 if ((pp->geom->flags & G_GEOM_WITHER) == 0)
 464                         retaste_wait = 1;
 465         }
 466         G_RAID3_DEBUG(2, "Access %s r%dw%de%d = %d", pp->name, -cp->acr,
 467             -cp->acw, -cp->ace, 0);
 468         if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
 469                 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
 470         if (retaste_wait) {
 471                 /*
 472                  * After retaste event was send (inside g_access()), we can send
 473                  * event to detach and destroy consumer.
 474                  * A class, which has consumer to the given provider connected
 475                  * will not receive retaste event for the provider.
 476                  * This is the way how I ignore retaste events when I close
 477                  * consumers opened for write: I detach and destroy consumer
 478                  * after retaste event is sent.
 479                  */
 480                 g_post_event(g_raid3_destroy_consumer, cp, M_WAITOK, NULL);
 481                 return;
 482         }
 483         G_RAID3_DEBUG(1, "Consumer %s destroyed.", pp->name);
 484         g_detach(cp);
 485         g_destroy_consumer(cp);
 486 }
 487
 488 static int
 489 g_raid3_connect_disk(struct g_raid3_disk *disk, struct g_provider *pp)
 490 {
 491         struct g_consumer *cp;
 492         int error;
 493
 494         g_topology_assert_not();
 495         KASSERT(disk->d_consumer == NULL,
 496             ("Disk already connected (device %s).", disk->d_softc->sc_name));
 497
 498         g_topology_lock();
 499         cp = g_new_consumer(disk->d_softc->sc_geom);
 500         error = g_attach(cp, pp);
 501         if (error != 0) {
 502                 g_destroy_consumer(cp);
 503                 g_topology_unlock();
 504                 return (error);
 505         }
 506         error = g_access(cp, 1, 1, 1);
 507                 g_topology_unlock();
 508         if (error != 0) {
 509                 g_detach(cp);
 510                 g_destroy_consumer(cp);
 511                 G_RAID3_DEBUG(0, "Cannot open consumer %s (error=%d).",
 512                     pp->name, error);
 513                 return (error);
 514         }
 515         disk->d_consumer = cp;
 516         disk->d_consumer->private = disk;
 517         disk->d_consumer->index = 0;
 518         G_RAID3_DEBUG(2, "Disk %s connected.", g_raid3_get_diskname(disk));
 519         return (0);
 520 }
 521
 522 static void
 523 g_raid3_disconnect_consumer(struct g_raid3_softc *sc, struct g_consumer *cp)
 524 {
 525
 526         g_topology_assert();
 527
 528         if (cp == NULL)
 529                 return;
 530         if (cp->provider != NULL)
 531                 g_raid3_kill_consumer(sc, cp);
 532         else
 533                 g_destroy_consumer(cp);
 534 }
 535
 536 /*
 537  * Initialize disk. This means allocate memory, create consumer, attach it
 538  * to the provider and open access (r1w1e1) to it.
 539  */
 540 static struct g_raid3_disk *
 541 g_raid3_init_disk(struct g_raid3_softc *sc, struct g_provider *pp,
 542     struct g_raid3_metadata *md, int *errorp)
 543 {
 544         struct g_raid3_disk *disk;
 545         int error;
 546
 547         disk = &sc->sc_disks[md->md_no];
 548         error = g_raid3_connect_disk(disk, pp);
 549         if (error != 0) {
 550                 if (errorp != NULL)
 551                         *errorp = error;
 552                 return (NULL);
 553         }
 554         disk->d_state = G_RAID3_DISK_STATE_NONE;
 555         disk->d_flags = md->md_dflags;
 556         if (md->md_provider[0] != '\0')
 557                 disk->d_flags |= G_RAID3_DISK_FLAG_HARDCODED;
 558         disk->d_sync.ds_consumer = NULL;
 559         disk->d_sync.ds_offset = md->md_sync_offset;
 560         disk->d_sync.ds_offset_done = md->md_sync_offset;
 561         disk->d_genid = md->md_genid;
 562         disk->d_sync.ds_syncid = md->md_syncid;
 563         if (errorp != NULL)
 564                 *errorp = 0;
 565         return (disk);
 566 }
 567
 568 static void
 569 g_raid3_destroy_disk(struct g_raid3_disk *disk)
 570 {
 571         struct g_raid3_softc *sc;
 572
 573         g_topology_assert_not();
 574         sc = disk->d_softc;
 575         sx_assert(&sc->sc_lock, SX_XLOCKED);
 576
 577         if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
 578                 return;
 579         g_raid3_event_cancel(disk);
 580         switch (disk->d_state) {
 581         case G_RAID3_DISK_STATE_SYNCHRONIZING:
 582                 if (sc->sc_syncdisk != NULL)
 583                         g_raid3_sync_stop(sc, 1);
 584                 /* FALLTHROUGH */
 585         case G_RAID3_DISK_STATE_NEW:
 586         case G_RAID3_DISK_STATE_STALE:
 587         case G_RAID3_DISK_STATE_ACTIVE:
 588                 g_topology_lock();
 589                 g_raid3_disconnect_consumer(sc, disk->d_consumer);
 590                 g_topology_unlock();
 591                 disk->d_consumer = NULL;
 592                 break;
 593         default:
 594                 KASSERT(0 == 1, ("Wrong disk state (%s, %s).",
 595                     g_raid3_get_diskname(disk),
 596                     g_raid3_disk_state2str(disk->d_state)));
 597         }
 598         disk->d_state = G_RAID3_DISK_STATE_NODISK;
 599 }
 600
 601 static void
 602 g_raid3_destroy_device(struct g_raid3_softc *sc)
 603 {
 604         struct g_raid3_event *ep;
 605         struct g_raid3_disk *disk;
 606         struct g_geom *gp;
 607         struct g_consumer *cp;
 608         u_int n;
 609
 610         g_topology_assert_not();
 611         sx_assert(&sc->sc_lock, SX_XLOCKED);
 612
 613         gp = sc->sc_geom;
 614         if (sc->sc_provider != NULL)
 615                 g_raid3_destroy_provider(sc);
 616         for (n = 0; n < sc->sc_ndisks; n++) {
 617                 disk = &sc->sc_disks[n];
 618                 if (disk->d_state != G_RAID3_DISK_STATE_NODISK) {
 619                         disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
 620                         g_raid3_update_metadata(disk);
 621                         g_raid3_destroy_disk(disk);
 622                 }
 623         }
 624         while ((ep = g_raid3_event_get(sc)) != NULL) {
 625                 g_raid3_event_remove(sc, ep);
 626                 if ((ep->e_flags & G_RAID3_EVENT_DONTWAIT) != 0)
 627                         g_raid3_event_free(ep);
 628                 else {
 629                         ep->e_error = ECANCELED;
 630                         ep->e_flags |= G_RAID3_EVENT_DONE;
 631                         G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, ep);
 632                         mtx_lock(&sc->sc_events_mtx);
 633                         wakeup(ep);
 634                         mtx_unlock(&sc->sc_events_mtx);
 635                 }
 636         }
 637         callout_drain(&sc->sc_callout);
 638         cp = LIST_FIRST(&sc->sc_sync.ds_geom->consumer);
 639         g_topology_lock();
 640         if (cp != NULL)
 641                 g_raid3_disconnect_consumer(sc, cp);
 642         g_wither_geom(sc->sc_sync.ds_geom, ENXIO);
 643         G_RAID3_DEBUG(0, "Device %s destroyed.", gp->name);
 644         g_wither_geom(gp, ENXIO);
 645         g_topology_unlock();
 646         if (!g_raid3_use_malloc) {
 647                 uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_64K].sz_zone);
 648                 uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_16K].sz_zone);
 649                 uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_4K].sz_zone);
 650         }
 651         mtx_destroy(&sc->sc_queue_mtx);
 652         mtx_destroy(&sc->sc_events_mtx);
 653         sx_xunlock(&sc->sc_lock);
 654         sx_destroy(&sc->sc_lock);
 655 }
 656
 657 static void
 658 g_raid3_orphan(struct g_consumer *cp)
 659 {
 660         struct g_raid3_disk *disk;
 661
 662         g_topology_assert();
 663
 664         disk = cp->private;
 665         if (disk == NULL)
 666                 return;
 667         disk->d_softc->sc_bump_id = G_RAID3_BUMP_SYNCID;
 668         g_raid3_event_send(disk, G_RAID3_DISK_STATE_DISCONNECTED,
 669             G_RAID3_EVENT_DONTWAIT);
 670 }
 671
 672 static int
 673 g_raid3_write_metadata(struct g_raid3_disk *disk, struct g_raid3_metadata *md)
 674 {
 675         struct g_raid3_softc *sc;
 676         struct g_consumer *cp;
 677         off_t offset, length;
 678         u_char *sector;
 679         int error = 0;
 680
 681         g_topology_assert_not();
 682         sc = disk->d_softc;
 683         sx_assert(&sc->sc_lock, SX_LOCKED);
 684
 685         cp = disk->d_consumer;
 686         KASSERT(cp != NULL, ("NULL consumer (%s).", sc->sc_name));
 687         KASSERT(cp->provider != NULL, ("NULL provider (%s).", sc->sc_name));
 688         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
 689             ("Consumer %s closed? (r%dw%de%d).", cp->provider->name, cp->acr,
 690             cp->acw, cp->ace));
 691         length = cp->provider->sectorsize;
 692         offset = cp->provider->mediasize - length;
 693         sector = malloc((size_t)length, M_RAID3, M_WAITOK | M_ZERO);
 694         if (md != NULL)
 695                 raid3_metadata_encode(md, sector);
 696         error = g_write_data(cp, offset, sector, length);
 697         free(sector, M_RAID3);
 698         if (error != 0) {
 699                 if ((disk->d_flags & G_RAID3_DISK_FLAG_BROKEN) == 0) {
 700                         G_RAID3_DEBUG(0, "Cannot write metadata on %s "
 701                             "(device=%s, error=%d).",
 702                             g_raid3_get_diskname(disk), sc->sc_name, error);
 703                         disk->d_flags |= G_RAID3_DISK_FLAG_BROKEN;
 704                 } else {
 705                         G_RAID3_DEBUG(1, "Cannot write metadata on %s "
 706                             "(device=%s, error=%d).",
 707                             g_raid3_get_diskname(disk), sc->sc_name, error);
 708                 }
 709                 if (g_raid3_disconnect_on_failure &&
 710                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
 711                         sc->sc_bump_id |= G_RAID3_BUMP_GENID;
 712                         g_raid3_event_send(disk,
 713                             G_RAID3_DISK_STATE_DISCONNECTED,
 714                             G_RAID3_EVENT_DONTWAIT);
 715                 }
 716         }
 717         return (error);
 718 }
 719
 720 int
 721 g_raid3_clear_metadata(struct g_raid3_disk *disk)
 722 {
 723         int error;
 724
 725         g_topology_assert_not();
 726         sx_assert(&disk->d_softc->sc_lock, SX_LOCKED);
 727
 728         error = g_raid3_write_metadata(disk, NULL);
 729         if (error == 0) {
 730                 G_RAID3_DEBUG(2, "Metadata on %s cleared.",
 731                     g_raid3_get_diskname(disk));
 732         } else {
 733                 G_RAID3_DEBUG(0,
 734                     "Cannot clear metadata on disk %s (error=%d).",
 735                     g_raid3_get_diskname(disk), error);
 736         }
 737         return (error);
 738 }
 739
 740 void
 741 g_raid3_fill_metadata(struct g_raid3_disk *disk, struct g_raid3_metadata *md)
 742 {
 743         struct g_raid3_softc *sc;
 744         struct g_provider *pp;
 745
 746         sc = disk->d_softc;
 747         strlcpy(md->md_magic, G_RAID3_MAGIC, sizeof(md->md_magic));
 748         md->md_version = G_RAID3_VERSION;
 749         strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 750         md->md_id = sc->sc_id;
 751         md->md_all = sc->sc_ndisks;
 752         md->md_genid = sc->sc_genid;
 753         md->md_mediasize = sc->sc_mediasize;
 754         md->md_sectorsize = sc->sc_sectorsize;
 755         md->md_mflags = (sc->sc_flags & G_RAID3_DEVICE_FLAG_MASK);
 756         md->md_no = disk->d_no;
 757         md->md_syncid = disk->d_sync.ds_syncid;
 758         md->md_dflags = (disk->d_flags & G_RAID3_DISK_FLAG_MASK);
 759         if (disk->d_state != G_RAID3_DISK_STATE_SYNCHRONIZING)
 760                 md->md_sync_offset = 0;
 761         else {
 762                 md->md_sync_offset =
 763                     disk->d_sync.ds_offset_done / (sc->sc_ndisks - 1);
 764         }
 765         if (disk->d_consumer != NULL && disk->d_consumer->provider != NULL)
 766                 pp = disk->d_consumer->provider;
 767         else
 768                 pp = NULL;
 769         if ((disk->d_flags & G_RAID3_DISK_FLAG_HARDCODED) != 0 && pp != NULL)
 770                 strlcpy(md->md_provider, pp->name, sizeof(md->md_provider));
 771         else
 772                 bzero(md->md_provider, sizeof(md->md_provider));
 773         if (pp != NULL)
 774                 md->md_provsize = pp->mediasize;
 775         else
 776                 md->md_provsize = 0;
 777 }
 778
 779 void
 780 g_raid3_update_metadata(struct g_raid3_disk *disk)
 781 {
 782         struct g_raid3_softc *sc;
 783         struct g_raid3_metadata md;
 784         int error;
 785
 786         g_topology_assert_not();
 787         sc = disk->d_softc;
 788         sx_assert(&sc->sc_lock, SX_LOCKED);
 789
 790         g_raid3_fill_metadata(disk, &md);
 791         error = g_raid3_write_metadata(disk, &md);
 792         if (error == 0) {
 793                 G_RAID3_DEBUG(2, "Metadata on %s updated.",
 794                     g_raid3_get_diskname(disk));
 795         } else {
 796                 G_RAID3_DEBUG(0,
 797                     "Cannot update metadata on disk %s (error=%d).",
 798                     g_raid3_get_diskname(disk), error);
 799         }
 800 }
 801
 802 static void
 803 g_raid3_bump_syncid(struct g_raid3_softc *sc)
 804 {
 805         struct g_raid3_disk *disk;
 806         u_int n;
 807
 808         g_topology_assert_not();
 809         sx_assert(&sc->sc_lock, SX_XLOCKED);
 810         KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) > 0,
 811             ("%s called with no active disks (device=%s).", __func__,
 812             sc->sc_name));
 813
 814         sc->sc_syncid++;
 815         G_RAID3_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
 816             sc->sc_syncid);
 817         for (n = 0; n < sc->sc_ndisks; n++) {
 818                 disk = &sc->sc_disks[n];
 819                 if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
 820                     disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
 821                         disk->d_sync.ds_syncid = sc->sc_syncid;
 822                         g_raid3_update_metadata(disk);
 823                 }
 824         }
 825 }
 826
 827 static void
 828 g_raid3_bump_genid(struct g_raid3_softc *sc)
 829 {
 830         struct g_raid3_disk *disk;
 831         u_int n;
 832
 833         g_topology_assert_not();
 834         sx_assert(&sc->sc_lock, SX_XLOCKED);
 835         KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) > 0,
 836             ("%s called with no active disks (device=%s).", __func__,
 837             sc->sc_name));
 838
 839         sc->sc_genid++;
 840         G_RAID3_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
 841             sc->sc_genid);
 842         for (n = 0; n < sc->sc_ndisks; n++) {
 843                 disk = &sc->sc_disks[n];
 844                 if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
 845                     disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
 846                         disk->d_genid = sc->sc_genid;
 847                         g_raid3_update_metadata(disk);
 848                 }
 849         }
 850 }
 851
 852 static int
 853 g_raid3_idle(struct g_raid3_softc *sc, int acw)
 854 {
 855         struct g_raid3_disk *disk;
 856         u_int i;
 857         int timeout;
 858
 859         g_topology_assert_not();
 860         sx_assert(&sc->sc_lock, SX_XLOCKED);
 861
 862         if (sc->sc_provider == NULL)
 863                 return (0);
 864         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) != 0)
 865                 return (0);
 866         if (sc->sc_idle)
 867                 return (0);
 868         if (sc->sc_writes > 0)
 869                 return (0);
 870         if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 871                 timeout = g_raid3_idletime - (time_uptime - sc->sc_last_write);
 872                 if (!g_raid3_shutdown && timeout > 0)
 873                         return (timeout);
 874         }
 875         sc->sc_idle = 1;
 876         for (i = 0; i < sc->sc_ndisks; i++) {
 877                 disk = &sc->sc_disks[i];
 878                 if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE)
 879                         continue;
 880                 G_RAID3_DEBUG(1, "Disk %s (device %s) marked as clean.",
 881                     g_raid3_get_diskname(disk), sc->sc_name);
 882                 disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
 883                 g_raid3_update_metadata(disk);
 884         }
 885         return (0);
 886 }
 887
 888 static void
 889 g_raid3_unidle(struct g_raid3_softc *sc)
 890 {
 891         struct g_raid3_disk *disk;
 892         u_int i;
 893
 894         g_topology_assert_not();
 895         sx_assert(&sc->sc_lock, SX_XLOCKED);
 896
 897         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) != 0)
 898                 return;
 899         sc->sc_idle = 0;
 900         sc->sc_last_write = time_uptime;
 901         for (i = 0; i < sc->sc_ndisks; i++) {
 902                 disk = &sc->sc_disks[i];
 903                 if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE)
 904                         continue;
 905                 G_RAID3_DEBUG(1, "Disk %s (device %s) marked as dirty.",
 906                     g_raid3_get_diskname(disk), sc->sc_name);
 907                 disk->d_flags |= G_RAID3_DISK_FLAG_DIRTY;
 908                 g_raid3_update_metadata(disk);
 909         }
 910 }
 911
 912 /*
 913  * Treat bio_driver1 field in parent bio as list head and field bio_caller1
 914  * in child bio as pointer to the next element on the list.
 915  */
 916 #define G_RAID3_HEAD_BIO(pbp)   (pbp)->bio_driver1
 917
 918 #define G_RAID3_NEXT_BIO(cbp)   (cbp)->bio_caller1
 919
 920 #define G_RAID3_FOREACH_BIO(pbp, bp)                                    \
 921         for ((bp) = G_RAID3_HEAD_BIO(pbp); (bp) != NULL;                \
 922             (bp) = G_RAID3_NEXT_BIO(bp))
 923
 924 #define G_RAID3_FOREACH_SAFE_BIO(pbp, bp, tmpbp)                        \
 925         for ((bp) = G_RAID3_HEAD_BIO(pbp);                              \
 926             (bp) != NULL && ((tmpbp) = G_RAID3_NEXT_BIO(bp), 1);        \
 927             (bp) = (tmpbp))
 928
 929 static void
 930 g_raid3_init_bio(struct bio *pbp)
 931 {
 932
 933         G_RAID3_HEAD_BIO(pbp) = NULL;
 934 }
 935
 936 static void
 937 g_raid3_remove_bio(struct bio *cbp)
 938 {
 939         struct bio *pbp, *bp;
 940
 941         pbp = cbp->bio_parent;
 942         if (G_RAID3_HEAD_BIO(pbp) == cbp)
 943                 G_RAID3_HEAD_BIO(pbp) = G_RAID3_NEXT_BIO(cbp);
 944         else {
 945                 G_RAID3_FOREACH_BIO(pbp, bp) {
 946                         if (G_RAID3_NEXT_BIO(bp) == cbp) {
 947                                 G_RAID3_NEXT_BIO(bp) = G_RAID3_NEXT_BIO(cbp);
 948                                 break;
 949                         }
 950                 }
 951         }
 952         G_RAID3_NEXT_BIO(cbp) = NULL;
 953 }
 954
 955 static void
 956 g_raid3_replace_bio(struct bio *sbp, struct bio *dbp)
 957 {
 958         struct bio *pbp, *bp;
 959
 960         g_raid3_remove_bio(sbp);
 961         pbp = dbp->bio_parent;
 962         G_RAID3_NEXT_BIO(sbp) = G_RAID3_NEXT_BIO(dbp);
 963         if (G_RAID3_HEAD_BIO(pbp) == dbp)
 964                 G_RAID3_HEAD_BIO(pbp) = sbp;
 965         else {
 966                 G_RAID3_FOREACH_BIO(pbp, bp) {
 967                         if (G_RAID3_NEXT_BIO(bp) == dbp) {
 968                                 G_RAID3_NEXT_BIO(bp) = sbp;
 969                                 break;
 970                         }
 971                 }
 972         }
 973         G_RAID3_NEXT_BIO(dbp) = NULL;
 974 }
 975
 976 static void
 977 g_raid3_destroy_bio(struct g_raid3_softc *sc, struct bio *cbp)
 978 {
 979         struct bio *bp, *pbp;
 980         size_t size;
 981
 982         pbp = cbp->bio_parent;
 983         pbp->bio_children--;
 984         KASSERT(cbp->bio_data != NULL, ("NULL bio_data"));
 985         size = pbp->bio_length / (sc->sc_ndisks - 1);
 986         g_raid3_free(sc, cbp->bio_data, size);
 987         if (G_RAID3_HEAD_BIO(pbp) == cbp) {
 988                 G_RAID3_HEAD_BIO(pbp) = G_RAID3_NEXT_BIO(cbp);
 989                 G_RAID3_NEXT_BIO(cbp) = NULL;
 990                 g_destroy_bio(cbp);
 991         } else {
 992                 G_RAID3_FOREACH_BIO(pbp, bp) {
 993                         if (G_RAID3_NEXT_BIO(bp) == cbp)
 994                                 break;
 995                 }
 996                 if (bp != NULL) {
 997                         KASSERT(G_RAID3_NEXT_BIO(bp) != NULL,
 998                             ("NULL bp->bio_driver1"));
 999                         G_RAID3_NEXT_BIO(bp) = G_RAID3_NEXT_BIO(cbp);
1000                         G_RAID3_NEXT_BIO(cbp) = NULL;
1001                 }
1002                 g_destroy_bio(cbp);
1003         }
1004 }
1005
1006 static struct bio *
1007 g_raid3_clone_bio(struct g_raid3_softc *sc, struct bio *pbp)
1008 {
1009         struct bio *bp, *cbp;
1010         size_t size;
1011         int memflag;
1012
1013         cbp = g_clone_bio(pbp);
1014         if (cbp == NULL)
1015                 return (NULL);
1016         size = pbp->bio_length / (sc->sc_ndisks - 1);
1017         if ((pbp->bio_cflags & G_RAID3_BIO_CFLAG_REGULAR) != 0)
1018                 memflag = M_WAITOK;
1019         else
1020                 memflag = M_NOWAIT;
1021         cbp->bio_data = g_raid3_alloc(sc, size, memflag);
1022         if (cbp->bio_data == NULL) {
1023                 pbp->bio_children--;
1024                 g_destroy_bio(cbp);
1025                 return (NULL);
1026         }
1027         G_RAID3_NEXT_BIO(cbp) = NULL;
1028         if (G_RAID3_HEAD_BIO(pbp) == NULL)
1029                 G_RAID3_HEAD_BIO(pbp) = cbp;
1030         else {
1031                 G_RAID3_FOREACH_BIO(pbp, bp) {
1032                         if (G_RAID3_NEXT_BIO(bp) == NULL) {
1033                                 G_RAID3_NEXT_BIO(bp) = cbp;
1034                                 break;
1035                         }
1036                 }
1037         }
1038         return (cbp);
1039 }
1040
1041 static void
1042 g_raid3_scatter(struct bio *pbp)
1043 {
1044         struct g_raid3_softc *sc;
1045         struct g_raid3_disk *disk;
1046         struct bio *bp, *cbp, *tmpbp;
1047         off_t atom, cadd, padd, left;
1048         int first;
1049
1050         sc = pbp->bio_to->geom->softc;
1051         bp = NULL;
1052         if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_NOPARITY) == 0) {
1053                 /*
1054                  * Find bio for which we should calculate data.
1055                  */
1056                 G_RAID3_FOREACH_BIO(pbp, cbp) {
1057                         if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_PARITY) != 0) {
1058                                 bp = cbp;
1059                                 break;
1060                         }
1061                 }
1062                 KASSERT(bp != NULL, ("NULL parity bio."));
1063         }
1064         atom = sc->sc_sectorsize / (sc->sc_ndisks - 1);
1065         cadd = padd = 0;
1066         for (left = pbp->bio_length; left > 0; left -= sc->sc_sectorsize) {
1067                 G_RAID3_FOREACH_BIO(pbp, cbp) {
1068                         if (cbp == bp)
1069                                 continue;
1070                         bcopy(pbp->bio_data + padd, cbp->bio_data + cadd, atom);
1071                         padd += atom;
1072                 }
1073                 cadd += atom;
1074         }
1075         if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_NOPARITY) == 0) {
1076                 /*
1077                  * Calculate parity.
1078                  */
1079                 first = 1;
1080                 G_RAID3_FOREACH_SAFE_BIO(pbp, cbp, tmpbp) {
1081                         if (cbp == bp)
1082                                 continue;
1083                         if (first) {
1084                                 bcopy(cbp->bio_data, bp->bio_data,
1085                                     bp->bio_length);
1086                                 first = 0;
1087                         } else {
1088                                 g_raid3_xor(cbp->bio_data, bp->bio_data,
1089                                     bp->bio_length);
1090                         }
1091                         if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_NODISK) != 0)
1092                                 g_raid3_destroy_bio(sc, cbp);
1093                 }
1094         }
1095         G_RAID3_FOREACH_SAFE_BIO(pbp, cbp, tmpbp) {
1096                 struct g_consumer *cp;
1097
1098                 disk = cbp->bio_caller2;
1099                 cp = disk->d_consumer;
1100                 cbp->bio_to = cp->provider;
1101                 G_RAID3_LOGREQ(3, cbp, "Sending request.");
1102                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1103                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1104                     cp->acr, cp->acw, cp->ace));
1105                 cp->index++;
1106                 sc->sc_writes++;
1107                 g_io_request(cbp, cp);
1108         }
1109 }
1110
1111 static void
1112 g_raid3_gather(struct bio *pbp)
1113 {
1114         struct g_raid3_softc *sc;
1115         struct g_raid3_disk *disk;
1116         struct bio *xbp, *fbp, *cbp;
1117         off_t atom, cadd, padd, left;
1118
1119         sc = pbp->bio_to->geom->softc;
1120         /*
1121          * Find bio for which we have to calculate data.
1122          * While going through this path, check if all requests
1123          * succeeded, if not, deny whole request.
1124          * If we're in COMPLETE mode, we allow one request to fail,
1125          * so if we find one, we're sending it to the parity consumer.
1126          * If there are more failed requests, we deny whole request.
1127          */
1128         xbp = fbp = NULL;
1129         G_RAID3_FOREACH_BIO(pbp, cbp) {
1130                 if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_PARITY) != 0) {
1131                         KASSERT(xbp == NULL, ("More than one parity bio."));
1132                         xbp = cbp;
1133                 }
1134                 if (cbp->bio_error == 0)
1135                         continue;
1136                 /*
1137                  * Found failed request.
1138                  */
1139                 if (fbp == NULL) {
1140                         if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_DEGRADED) != 0) {
1141                                 /*
1142                                  * We are already in degraded mode, so we can't
1143                                  * accept any failures.
1144                                  */
1145                                 if (pbp->bio_error == 0)
1146                                         pbp->bio_error = cbp->bio_error;
1147                         } else {
1148                                 fbp = cbp;
1149                         }
1150                 } else {
1151                         /*
1152                          * Next failed request, that's too many.
1153                          */
1154                         if (pbp->bio_error == 0)
1155                                 pbp->bio_error = fbp->bio_error;
1156                 }
1157                 disk = cbp->bio_caller2;
1158                 if (disk == NULL)
1159                         continue;
1160                 if ((disk->d_flags & G_RAID3_DISK_FLAG_BROKEN) == 0) {
1161                         disk->d_flags |= G_RAID3_DISK_FLAG_BROKEN;
1162                         G_RAID3_LOGREQ(0, cbp, "Request failed (error=%d).",
1163                             cbp->bio_error);
1164                 } else {
1165                         G_RAID3_LOGREQ(1, cbp, "Request failed (error=%d).",
1166                             cbp->bio_error);
1167                 }
1168                 if (g_raid3_disconnect_on_failure &&
1169                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
1170                         sc->sc_bump_id |= G_RAID3_BUMP_GENID;
1171                         g_raid3_event_send(disk,
1172                             G_RAID3_DISK_STATE_DISCONNECTED,
1173                             G_RAID3_EVENT_DONTWAIT);
1174                 }
1175         }
1176         if (pbp->bio_error != 0)
1177                 goto finish;
1178         if (fbp != NULL && (pbp->bio_pflags & G_RAID3_BIO_PFLAG_VERIFY) != 0) {
1179                 pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_VERIFY;
1180                 if (xbp != fbp)
1181                         g_raid3_replace_bio(xbp, fbp);
1182                 g_raid3_destroy_bio(sc, fbp);
1183         } else if (fbp != NULL) {
1184                 struct g_consumer *cp;
1185
1186                 /*
1187                  * One request failed, so send the same request to
1188                  * the parity consumer.
1189                  */
1190                 disk = pbp->bio_driver2;
1191                 if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE) {
1192                         pbp->bio_error = fbp->bio_error;
1193                         goto finish;
1194                 }
1195                 pbp->bio_pflags |= G_RAID3_BIO_PFLAG_DEGRADED;
1196                 pbp->bio_inbed--;
1197                 fbp->bio_flags &= ~(BIO_DONE | BIO_ERROR);
1198                 if (disk->d_no == sc->sc_ndisks - 1)
1199                         fbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
1200                 fbp->bio_error = 0;
1201                 fbp->bio_completed = 0;
1202                 fbp->bio_children = 0;
1203                 fbp->bio_inbed = 0;
1204                 cp = disk->d_consumer;
1205                 fbp->bio_caller2 = disk;
1206                 fbp->bio_to = cp->provider;
1207                 G_RAID3_LOGREQ(3, fbp, "Sending request (recover).");
1208                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1209                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1210                     cp->acr, cp->acw, cp->ace));
1211                 cp->index++;
1212                 g_io_request(fbp, cp);
1213                 return;
1214         }
1215         if (xbp != NULL) {
1216                 /*
1217                  * Calculate parity.
1218                  */
1219                 G_RAID3_FOREACH_BIO(pbp, cbp) {
1220                         if ((cbp->bio_cflags & G_RAID3_BIO_CFLAG_PARITY) != 0)
1221                                 continue;
1222                         g_raid3_xor(cbp->bio_data, xbp->bio_data,
1223                             xbp->bio_length);
1224                 }
1225                 xbp->bio_cflags &= ~G_RAID3_BIO_CFLAG_PARITY;
1226                 if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_VERIFY) != 0) {
1227                         if (!g_raid3_is_zero(xbp)) {
1228                                 g_raid3_parity_mismatch++;
1229                                 pbp->bio_error = EIO;
1230                                 goto finish;
1231                         }
1232                         g_raid3_destroy_bio(sc, xbp);
1233                 }
1234         }
1235         atom = sc->sc_sectorsize / (sc->sc_ndisks - 1);
1236         cadd = padd = 0;
1237         for (left = pbp->bio_length; left > 0; left -= sc->sc_sectorsize) {
1238                 G_RAID3_FOREACH_BIO(pbp, cbp) {
1239                         bcopy(cbp->bio_data + cadd, pbp->bio_data + padd, atom);
1240                         pbp->bio_completed += atom;
1241                         padd += atom;
1242                 }
1243                 cadd += atom;
1244         }
1245 finish:
1246         if (pbp->bio_error == 0)
1247                 G_RAID3_LOGREQ(3, pbp, "Request finished.");
1248         else {
1249                 if ((pbp->bio_pflags & G_RAID3_BIO_PFLAG_VERIFY) != 0)
1250                         G_RAID3_LOGREQ(1, pbp, "Verification error.");
1251                 else
1252                         G_RAID3_LOGREQ(0, pbp, "Request failed.");
1253         }
1254         pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_MASK;
1255         while ((cbp = G_RAID3_HEAD_BIO(pbp)) != NULL)
1256                 g_raid3_destroy_bio(sc, cbp);
1257         g_io_deliver(pbp, pbp->bio_error);
1258 }
1259
1260 static void
1261 g_raid3_done(struct bio *bp)
1262 {
1263         struct g_raid3_softc *sc;
1264
1265         sc = bp->bio_from->geom->softc;
1266         bp->bio_cflags |= G_RAID3_BIO_CFLAG_REGULAR;
1267         G_RAID3_LOGREQ(3, bp, "Regular request done (error=%d).", bp->bio_error);
1268         mtx_lock(&sc->sc_queue_mtx);
1269         bioq_insert_head(&sc->sc_queue, bp);
1270         mtx_unlock(&sc->sc_queue_mtx);
1271         wakeup(sc);
1272         wakeup(&sc->sc_queue);
1273 }
1274
1275 static void
1276 g_raid3_regular_request(struct bio *cbp)
1277 {
1278         struct g_raid3_softc *sc;
1279         struct g_raid3_disk *disk;
1280         struct bio *pbp;
1281
1282         g_topology_assert_not();
1283
1284         pbp = cbp->bio_parent;
1285         sc = pbp->bio_to->geom->softc;
1286         cbp->bio_from->index--;
1287         if (cbp->bio_cmd == BIO_WRITE)
1288                 sc->sc_writes--;
1289         disk = cbp->bio_from->private;
1290         if (disk == NULL) {
1291                 g_topology_lock();
1292                 g_raid3_kill_consumer(sc, cbp->bio_from);
1293                 g_topology_unlock();
1294         }
1295
1296         G_RAID3_LOGREQ(3, cbp, "Request finished.");
1297         pbp->bio_inbed++;
1298         KASSERT(pbp->bio_inbed <= pbp->bio_children,
1299             ("bio_inbed (%u) is bigger than bio_children (%u).", pbp->bio_inbed,
1300             pbp->bio_children));
1301         if (pbp->bio_inbed != pbp->bio_children)
1302                 return;
1303         switch (pbp->bio_cmd) {
1304         case BIO_READ:
1305                 g_raid3_gather(pbp);
1306                 break;
1307         case BIO_WRITE:
1308         case BIO_DELETE:
1309             {
1310                 int error = 0;
1311
1312                 pbp->bio_completed = pbp->bio_length;
1313                 while ((cbp = G_RAID3_HEAD_BIO(pbp)) != NULL) {
1314                         if (cbp->bio_error == 0) {
1315                                 g_raid3_destroy_bio(sc, cbp);
1316                                 continue;
1317                         }
1318
1319                         if (error == 0)
1320                                 error = cbp->bio_error;
1321                         else if (pbp->bio_error == 0) {
1322                                 /*
1323                                  * Next failed request, that's too many.
1324                                  */
1325                                 pbp->bio_error = error;
1326                         }
1327
1328                         disk = cbp->bio_caller2;
1329                         if (disk == NULL) {
1330                                 g_raid3_destroy_bio(sc, cbp);
1331                                 continue;
1332                         }
1333
1334                         if ((disk->d_flags & G_RAID3_DISK_FLAG_BROKEN) == 0) {
1335                                 disk->d_flags |= G_RAID3_DISK_FLAG_BROKEN;
1336                                 G_RAID3_LOGREQ(0, cbp,
1337                                     "Request failed (error=%d).",
1338                                     cbp->bio_error);
1339                         } else {
1340                                 G_RAID3_LOGREQ(1, cbp,
1341                                     "Request failed (error=%d).",
1342                                     cbp->bio_error);
1343                         }
1344                         if (g_raid3_disconnect_on_failure &&
1345                             sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
1346                                 sc->sc_bump_id |= G_RAID3_BUMP_GENID;
1347                                 g_raid3_event_send(disk,
1348                                     G_RAID3_DISK_STATE_DISCONNECTED,
1349                                     G_RAID3_EVENT_DONTWAIT);
1350                         }
1351                         g_raid3_destroy_bio(sc, cbp);
1352                 }
1353                 if (pbp->bio_error == 0)
1354                         G_RAID3_LOGREQ(3, pbp, "Request finished.");
1355                 else
1356                         G_RAID3_LOGREQ(0, pbp, "Request failed.");
1357                 pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_DEGRADED;
1358                 pbp->bio_pflags &= ~G_RAID3_BIO_PFLAG_NOPARITY;
1359                 bioq_remove(&sc->sc_inflight, pbp);
1360                 /* Release delayed sync requests if possible. */
1361                 g_raid3_sync_release(sc);
1362                 g_io_deliver(pbp, pbp->bio_error);
1363                 break;
1364             }
1365         }
1366 }
1367
1368 static void
1369 g_raid3_sync_done(struct bio *bp)
1370 {
1371         struct g_raid3_softc *sc;
1372
1373         G_RAID3_LOGREQ(3, bp, "Synchronization request delivered.");
1374         sc = bp->bio_from->geom->softc;
1375         bp->bio_cflags |= G_RAID3_BIO_CFLAG_SYNC;
1376         mtx_lock(&sc->sc_queue_mtx);
1377         bioq_insert_head(&sc->sc_queue, bp);
1378         mtx_unlock(&sc->sc_queue_mtx);
1379         wakeup(sc);
1380         wakeup(&sc->sc_queue);
1381 }
1382
1383 static void
1384 g_raid3_flush(struct g_raid3_softc *sc, struct bio *bp)
1385 {
1386         struct bio_queue_head queue;
1387         struct g_raid3_disk *disk;
1388         struct g_consumer *cp;
1389         struct bio *cbp;
1390         u_int i;
1391
1392         bioq_init(&queue);
1393         for (i = 0; i < sc->sc_ndisks; i++) {
1394                 disk = &sc->sc_disks[i];
1395                 if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE)
1396                         continue;
1397                 cbp = g_clone_bio(bp);
1398                 if (cbp == NULL) {
1399                         for (cbp = bioq_first(&queue); cbp != NULL;
1400                             cbp = bioq_first(&queue)) {
1401                                 bioq_remove(&queue, cbp);
1402                                 g_destroy_bio(cbp);
1403                         }
1404                         if (bp->bio_error == 0)
1405                                 bp->bio_error = ENOMEM;
1406                         g_io_deliver(bp, bp->bio_error);
1407                         return;
1408                 }
1409                 bioq_insert_tail(&queue, cbp);
1410                 cbp->bio_done = g_std_done;
1411                 cbp->bio_caller1 = disk;
1412                 cbp->bio_to = disk->d_consumer->provider;
1413         }
1414         for (cbp = bioq_first(&queue); cbp != NULL; cbp = bioq_first(&queue)) {
1415                 bioq_remove(&queue, cbp);
1416                 G_RAID3_LOGREQ(3, cbp, "Sending request.");
1417                 disk = cbp->bio_caller1;
1418                 cbp->bio_caller1 = NULL;
1419                 cp = disk->d_consumer;
1420                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1421                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1422                     cp->acr, cp->acw, cp->ace));
1423                 g_io_request(cbp, disk->d_consumer);
1424         }
1425 }
1426
1427 static void
1428 g_raid3_start(struct bio *bp)
1429 {
1430         struct g_raid3_softc *sc;
1431
1432         sc = bp->bio_to->geom->softc;
1433         /*
1434          * If sc == NULL or there are no valid disks, provider's error
1435          * should be set and g_raid3_start() should not be called at all.
1436          */
1437         KASSERT(sc != NULL && (sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
1438             sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE),
1439             ("Provider's error should be set (error=%d)(device=%s).",
1440             bp->bio_to->error, bp->bio_to->name));
1441         G_RAID3_LOGREQ(3, bp, "Request received.");
1442
1443         switch (bp->bio_cmd) {
1444         case BIO_READ:
1445         case BIO_WRITE:
1446         case BIO_DELETE:
1447                 break;
1448         case BIO_FLUSH:
1449                 g_raid3_flush(sc, bp);
1450                 return;
1451         case BIO_GETATTR:
1452         default:
1453                 g_io_deliver(bp, EOPNOTSUPP);
1454                 return;
1455         }
1456         mtx_lock(&sc->sc_queue_mtx);
1457         bioq_insert_tail(&sc->sc_queue, bp);
1458         mtx_unlock(&sc->sc_queue_mtx);
1459         G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, sc);
1460         wakeup(sc);
1461 }
1462
1463 /*
1464  * Return TRUE if the given request is colliding with a in-progress
1465  * synchronization request.
1466  */
1467 static int
1468 g_raid3_sync_collision(struct g_raid3_softc *sc, struct bio *bp)
1469 {
1470         struct g_raid3_disk *disk;
1471         struct bio *sbp;
1472         off_t rstart, rend, sstart, send;
1473         int i;
1474
1475         disk = sc->sc_syncdisk;
1476         if (disk == NULL)
1477                 return (0);
1478         rstart = bp->bio_offset;
1479         rend = bp->bio_offset + bp->bio_length;
1480         for (i = 0; i < g_raid3_syncreqs; i++) {
1481                 sbp = disk->d_sync.ds_bios[i];
1482                 if (sbp == NULL)
1483                         continue;
1484                 sstart = sbp->bio_offset;
1485                 send = sbp->bio_length;
1486                 if (sbp->bio_cmd == BIO_WRITE) {
1487                         sstart *= sc->sc_ndisks - 1;
1488                         send *= sc->sc_ndisks - 1;
1489                 }
1490                 send += sstart;
1491                 if (rend > sstart && rstart < send)
1492                         return (1);
1493         }
1494         return (0);
1495 }
1496
1497 /*
1498  * Return TRUE if the given sync request is colliding with a in-progress regular
1499  * request.
1500  */
1501 static int
1502 g_raid3_regular_collision(struct g_raid3_softc *sc, struct bio *sbp)
1503 {
1504         off_t rstart, rend, sstart, send;
1505         struct bio *bp;
1506
1507         if (sc->sc_syncdisk == NULL)
1508                 return (0);
1509         sstart = sbp->bio_offset;
1510         send = sstart + sbp->bio_length;
1511         TAILQ_FOREACH(bp, &sc->sc_inflight.queue, bio_queue) {
1512                 rstart = bp->bio_offset;
1513                 rend = bp->bio_offset + bp->bio_length;
1514                 if (rend > sstart && rstart < send)
1515                         return (1);
1516         }
1517         return (0);
1518 }
1519
1520 /*
1521  * Puts request onto delayed queue.
1522  */
1523 static void
1524 g_raid3_regular_delay(struct g_raid3_softc *sc, struct bio *bp)
1525 {
1526
1527         G_RAID3_LOGREQ(2, bp, "Delaying request.");
1528         bioq_insert_head(&sc->sc_regular_delayed, bp);
1529 }
1530
1531 /*
1532  * Puts synchronization request onto delayed queue.
1533  */
1534 static void
1535 g_raid3_sync_delay(struct g_raid3_softc *sc, struct bio *bp)
1536 {
1537
1538         G_RAID3_LOGREQ(2, bp, "Delaying synchronization request.");
1539         bioq_insert_tail(&sc->sc_sync_delayed, bp);
1540 }
1541
1542 /*
1543  * Releases delayed regular requests which don't collide anymore with sync
1544  * requests.
1545  */
1546 static void
1547 g_raid3_regular_release(struct g_raid3_softc *sc)
1548 {
1549         struct bio *bp, *bp2;
1550
1551         TAILQ_FOREACH_SAFE(bp, &sc->sc_regular_delayed.queue, bio_queue, bp2) {
1552                 if (g_raid3_sync_collision(sc, bp))
1553                         continue;
1554                 bioq_remove(&sc->sc_regular_delayed, bp);
1555                 G_RAID3_LOGREQ(2, bp, "Releasing delayed request (%p).", bp);
1556                 mtx_lock(&sc->sc_queue_mtx);
1557                 bioq_insert_head(&sc->sc_queue, bp);
1558 #if 0
1559                 /*
1560                  * wakeup() is not needed, because this function is called from
1561                  * the worker thread.
1562                  */
1563                 wakeup(&sc->sc_queue);
1564 #endif
1565                 mtx_unlock(&sc->sc_queue_mtx);
1566         }
1567 }
1568
1569 /*
1570  * Releases delayed sync requests which don't collide anymore with regular
1571  * requests.
1572  */
1573 static void
1574 g_raid3_sync_release(struct g_raid3_softc *sc)
1575 {
1576         struct bio *bp, *bp2;
1577
1578         TAILQ_FOREACH_SAFE(bp, &sc->sc_sync_delayed.queue, bio_queue, bp2) {
1579                 if (g_raid3_regular_collision(sc, bp))
1580                         continue;
1581                 bioq_remove(&sc->sc_sync_delayed, bp);
1582                 G_RAID3_LOGREQ(2, bp,
1583                     "Releasing delayed synchronization request.");
1584                 g_io_request(bp, bp->bio_from);
1585         }
1586 }
1587
1588 /*
1589  * Handle synchronization requests.
1590  * Every synchronization request is two-steps process: first, READ request is
1591  * send to active provider and then WRITE request (with read data) to the provider
1592  * being synchronized. When WRITE is finished, new synchronization request is
1593  * send.
1594  */
1595 static void
1596 g_raid3_sync_request(struct bio *bp)
1597 {
1598         struct g_raid3_softc *sc;
1599         struct g_raid3_disk *disk;
1600
1601         bp->bio_from->index--;
1602         sc = bp->bio_from->geom->softc;
1603         disk = bp->bio_from->private;
1604         if (disk == NULL) {
1605                 sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
1606                 g_topology_lock();
1607                 g_raid3_kill_consumer(sc, bp->bio_from);
1608                 g_topology_unlock();
1609                 free(bp->bio_data, M_RAID3);
1610                 g_destroy_bio(bp);
1611                 sx_xlock(&sc->sc_lock);
1612                 return;
1613         }
1614
1615         /*
1616          * Synchronization request.
1617          */
1618         switch (bp->bio_cmd) {
1619         case BIO_READ:
1620             {
1621                 struct g_consumer *cp;
1622                 u_char *dst, *src;
1623                 off_t left;
1624                 u_int atom;
1625
1626                 if (bp->bio_error != 0) {
1627                         G_RAID3_LOGREQ(0, bp,
1628                             "Synchronization request failed (error=%d).",
1629                             bp->bio_error);
1630                         g_destroy_bio(bp);
1631                         return;
1632                 }
1633                 G_RAID3_LOGREQ(3, bp, "Synchronization request finished.");
1634                 atom = sc->sc_sectorsize / (sc->sc_ndisks - 1);
1635                 dst = src = bp->bio_data;
1636                 if (disk->d_no == sc->sc_ndisks - 1) {
1637                         u_int n;
1638
1639                         /* Parity component. */
1640                         for (left = bp->bio_length; left > 0;
1641                             left -= sc->sc_sectorsize) {
1642                                 bcopy(src, dst, atom);
1643                                 src += atom;
1644                                 for (n = 1; n < sc->sc_ndisks - 1; n++) {
1645                                         g_raid3_xor(src, dst, atom);
1646                                         src += atom;
1647                                 }
1648                                 dst += atom;
1649                         }
1650                 } else {
1651                         /* Regular component. */
1652                         src += atom * disk->d_no;
1653                         for (left = bp->bio_length; left > 0;
1654                             left -= sc->sc_sectorsize) {
1655                                 bcopy(src, dst, atom);
1656                                 src += sc->sc_sectorsize;
1657                                 dst += atom;
1658                         }
1659                 }
1660                 bp->bio_driver1 = bp->bio_driver2 = NULL;
1661                 bp->bio_pflags = 0;
1662                 bp->bio_offset /= sc->sc_ndisks - 1;
1663                 bp->bio_length /= sc->sc_ndisks - 1;
1664                 bp->bio_cmd = BIO_WRITE;
1665                 bp->bio_cflags = 0;
1666                 bp->bio_children = bp->bio_inbed = 0;
1667                 cp = disk->d_consumer;
1668                 KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1669                     ("Consumer %s not opened (r%dw%de%d).", cp->provider->name,
1670                     cp->acr, cp->acw, cp->ace));
1671                 cp->index++;
1672                 g_io_request(bp, cp);
1673                 return;
1674             }
1675         case BIO_WRITE:
1676             {
1677                 struct g_raid3_disk_sync *sync;
1678                 off_t boffset, moffset;
1679                 void *data;
1680                 int i;
1681
1682                 if (bp->bio_error != 0) {
1683                         G_RAID3_LOGREQ(0, bp,
1684                             "Synchronization request failed (error=%d).",
1685                             bp->bio_error);
1686                         g_destroy_bio(bp);
1687                         sc->sc_bump_id |= G_RAID3_BUMP_GENID;
1688                         g_raid3_event_send(disk,
1689                             G_RAID3_DISK_STATE_DISCONNECTED,
1690                             G_RAID3_EVENT_DONTWAIT);
1691                         return;
1692                 }
1693                 G_RAID3_LOGREQ(3, bp, "Synchronization request finished.");
1694                 sync = &disk->d_sync;
1695                 if (sync->ds_offset == sc->sc_mediasize / (sc->sc_ndisks - 1) ||
1696                     sync->ds_consumer == NULL ||
1697                     (sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
1698                         /* Don't send more synchronization requests. */
1699                         sync->ds_inflight--;
1700                         if (sync->ds_bios != NULL) {
1701                                 i = (int)(uintptr_t)bp->bio_caller1;
1702                                 sync->ds_bios[i] = NULL;
1703                         }
1704                         free(bp->bio_data, M_RAID3);
1705                         g_destroy_bio(bp);
1706                         if (sync->ds_inflight > 0)
1707                                 return;
1708                         if (sync->ds_consumer == NULL ||
1709                             (sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
1710                                 return;
1711                         }
1712                         /*
1713                          * Disk up-to-date, activate it.
1714                          */
1715                         g_raid3_event_send(disk, G_RAID3_DISK_STATE_ACTIVE,
1716                             G_RAID3_EVENT_DONTWAIT);
1717                         return;
1718                 }
1719
1720                 /* Send next synchronization request. */
1721                 data = bp->bio_data;
1722                 g_reset_bio(bp);
1723                 bp->bio_cmd = BIO_READ;
1724                 bp->bio_offset = sync->ds_offset * (sc->sc_ndisks - 1);
1725                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
1726                 sync->ds_offset += bp->bio_length / (sc->sc_ndisks - 1);
1727                 bp->bio_done = g_raid3_sync_done;
1728                 bp->bio_data = data;
1729                 bp->bio_from = sync->ds_consumer;
1730                 bp->bio_to = sc->sc_provider;
1731                 G_RAID3_LOGREQ(3, bp, "Sending synchronization request.");
1732                 sync->ds_consumer->index++;
1733                 /*
1734                  * Delay the request if it is colliding with a regular request.
1735                  */
1736                 if (g_raid3_regular_collision(sc, bp))
1737                         g_raid3_sync_delay(sc, bp);
1738                 else
1739                         g_io_request(bp, sync->ds_consumer);
1740
1741                 /* Release delayed requests if possible. */
1742                 g_raid3_regular_release(sc);
1743
1744                 /* Find the smallest offset. */
1745                 moffset = sc->sc_mediasize;
1746                 for (i = 0; i < g_raid3_syncreqs; i++) {
1747                         bp = sync->ds_bios[i];
1748                         boffset = bp->bio_offset;
1749                         if (bp->bio_cmd == BIO_WRITE)
1750                                 boffset *= sc->sc_ndisks - 1;
1751                         if (boffset < moffset)
1752                                 moffset = boffset;
1753                 }
1754                 if (sync->ds_offset_done + (MAXPHYS * 100) < moffset) {
1755                         /* Update offset_done on every 100 blocks. */
1756                         sync->ds_offset_done = moffset;
1757                         g_raid3_update_metadata(disk);
1758                 }
1759                 return;
1760             }
1761         default:
1762                 KASSERT(1 == 0, ("Invalid command here: %u (device=%s)",
1763                     bp->bio_cmd, sc->sc_name));
1764                 break;
1765         }
1766 }
1767
1768 static int
1769 g_raid3_register_request(struct bio *pbp)
1770 {
1771         struct g_raid3_softc *sc;
1772         struct g_raid3_disk *disk;
1773         struct g_consumer *cp;
1774         struct bio *cbp, *tmpbp;
1775         off_t offset, length;
1776         u_int n, ndisks;
1777         int round_robin, verify;
1778
1779         ndisks = 0;
1780         sc = pbp->bio_to->geom->softc;
1781         if ((pbp->bio_cflags & G_RAID3_BIO_CFLAG_REGSYNC) != 0 &&
1782             sc->sc_syncdisk == NULL) {
1783                 g_io_deliver(pbp, EIO);
1784                 return (0);
1785         }
1786         g_raid3_init_bio(pbp);
1787         length = pbp->bio_length / (sc->sc_ndisks - 1);
1788         offset = pbp->bio_offset / (sc->sc_ndisks - 1);
1789         round_robin = verify = 0;
1790         switch (pbp->bio_cmd) {
1791         case BIO_READ:
1792                 if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_VERIFY) != 0 &&
1793                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
1794                         pbp->bio_pflags |= G_RAID3_BIO_PFLAG_VERIFY;
1795                         verify = 1;
1796                         ndisks = sc->sc_ndisks;
1797                 } else {
1798                         verify = 0;
1799                         ndisks = sc->sc_ndisks - 1;
1800                 }
1801                 if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_ROUND_ROBIN) != 0 &&
1802                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
1803                         round_robin = 1;
1804                 } else {
1805                         round_robin = 0;
1806                 }
1807                 KASSERT(!round_robin || !verify,
1808                     ("ROUND-ROBIN and VERIFY are mutually exclusive."));
1809                 pbp->bio_driver2 = &sc->sc_disks[sc->sc_ndisks - 1];
1810                 break;
1811         case BIO_WRITE:
1812         case BIO_DELETE:
1813                 /*
1814                  * Delay the request if it is colliding with a synchronization
1815                  * request.
1816                  */
1817                 if (g_raid3_sync_collision(sc, pbp)) {
1818                         g_raid3_regular_delay(sc, pbp);
1819                         return (0);
1820                 }
1821
1822                 if (sc->sc_idle)
1823                         g_raid3_unidle(sc);
1824                 else
1825                         sc->sc_last_write = time_uptime;
1826
1827                 ndisks = sc->sc_ndisks;
1828                 break;
1829         }
1830         for (n = 0; n < ndisks; n++) {
1831                 disk = &sc->sc_disks[n];
1832                 cbp = g_raid3_clone_bio(sc, pbp);
1833                 if (cbp == NULL) {
1834                         while ((cbp = G_RAID3_HEAD_BIO(pbp)) != NULL)
1835                                 g_raid3_destroy_bio(sc, cbp);
1836                         /*
1837                          * To prevent deadlock, we must run back up
1838                          * with the ENOMEM for failed requests of any
1839                          * of our consumers.  Our own sync requests
1840                          * can stick around, as they are finite.
1841                          */
1842                         if ((pbp->bio_cflags &
1843                             G_RAID3_BIO_CFLAG_REGULAR) != 0) {
1844                                 g_io_deliver(pbp, ENOMEM);
1845                                 return (0);
1846                         }
1847                         return (ENOMEM);
1848                 }
1849                 cbp->bio_offset = offset;
1850                 cbp->bio_length = length;
1851                 cbp->bio_done = g_raid3_done;
1852                 switch (pbp->bio_cmd) {
1853                 case BIO_READ:
1854                         if (disk->d_state != G_RAID3_DISK_STATE_ACTIVE) {
1855                                 /*
1856                                  * Replace invalid component with the parity
1857                                  * component.
1858                                  */
1859                                 disk = &sc->sc_disks[sc->sc_ndisks - 1];
1860                                 cbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
1861                                 pbp->bio_pflags |= G_RAID3_BIO_PFLAG_DEGRADED;
1862                         } else if (round_robin &&
1863                             disk->d_no == sc->sc_round_robin) {
1864                                 /*
1865                                  * In round-robin mode skip one data component
1866                                  * and use parity component when reading.
1867                                  */
1868                                 pbp->bio_driver2 = disk;
1869                                 disk = &sc->sc_disks[sc->sc_ndisks - 1];
1870                                 cbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
1871                                 sc->sc_round_robin++;
1872                                 round_robin = 0;
1873                         } else if (verify && disk->d_no == sc->sc_ndisks - 1) {
1874                                 cbp->bio_cflags |= G_RAID3_BIO_CFLAG_PARITY;
1875                         }
1876                         break;
1877                 case BIO_WRITE:
1878                 case BIO_DELETE:
1879                         if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
1880                             disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
1881                                 if (n == ndisks - 1) {
1882                                         /*
1883                                          * Active parity component, mark it as such.
1884                                          */
1885                                         cbp->bio_cflags |=
1886                                             G_RAID3_BIO_CFLAG_PARITY;
1887                                 }
1888                         } else {
1889                                 pbp->bio_pflags |= G_RAID3_BIO_PFLAG_DEGRADED;
1890                                 if (n == ndisks - 1) {
1891                                         /*
1892                                          * Parity component is not connected,
1893                                          * so destroy its request.
1894                                          */
1895                                         pbp->bio_pflags |=
1896                                             G_RAID3_BIO_PFLAG_NOPARITY;
1897                                         g_raid3_destroy_bio(sc, cbp);
1898                                         cbp = NULL;
1899                                 } else {
1900                                         cbp->bio_cflags |=
1901                                             G_RAID3_BIO_CFLAG_NODISK;
1902                                         disk = NULL;
1903                                 }
1904                         }
1905                         break;
1906                 }
1907                 if (cbp != NULL)
1908                         cbp->bio_caller2 = disk;
1909         }
1910         switch (pbp->bio_cmd) {
1911         case BIO_READ:
1912                 if (round_robin) {
1913                         /*
1914                          * If we are in round-robin mode and 'round_robin' is
1915                          * still 1, it means, that we skipped parity component
1916                          * for this read and must reset sc_round_robin field.
1917                          */
1918                         sc->sc_round_robin = 0;
1919                 }
1920                 G_RAID3_FOREACH_SAFE_BIO(pbp, cbp, tmpbp) {
1921                         disk = cbp->bio_caller2;
1922                         cp = disk->d_consumer;
1923                         cbp->bio_to = cp->provider;
1924                         G_RAID3_LOGREQ(3, cbp, "Sending request.");
1925                         KASSERT(cp->acr >= 1 && cp->acw >= 1 && cp->ace >= 1,
1926                             ("Consumer %s not opened (r%dw%de%d).",
1927                             cp->provider->name, cp->acr, cp->acw, cp->ace));
1928                         cp->index++;
1929                         g_io_request(cbp, cp);
1930                 }
1931                 break;
1932         case BIO_WRITE:
1933         case BIO_DELETE:
1934                 /*
1935                  * Put request onto inflight queue, so we can check if new
1936                  * synchronization requests don't collide with it.
1937                  */
1938                 bioq_insert_tail(&sc->sc_inflight, pbp);
1939
1940                 /*
1941                  * Bump syncid on first write.
1942                  */
1943                 if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID) != 0) {
1944                         sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
1945                         g_raid3_bump_syncid(sc);
1946                 }
1947                 g_raid3_scatter(pbp);
1948                 break;
1949         }
1950         return (0);
1951 }
1952
1953 static int
1954 g_raid3_can_destroy(struct g_raid3_softc *sc)
1955 {
1956         struct g_geom *gp;
1957         struct g_consumer *cp;
1958
1959         g_topology_assert();
1960         gp = sc->sc_geom;
1961         if (gp->softc == NULL)
1962                 return (1);
1963         LIST_FOREACH(cp, &gp->consumer, consumer) {
1964                 if (g_raid3_is_busy(sc, cp))
1965                         return (0);
1966         }
1967         gp = sc->sc_sync.ds_geom;
1968         LIST_FOREACH(cp, &gp->consumer, consumer) {
1969                 if (g_raid3_is_busy(sc, cp))
1970                         return (0);
1971         }
1972         G_RAID3_DEBUG(2, "No I/O requests for %s, it can be destroyed.",
1973             sc->sc_name);
1974         return (1);
1975 }
1976
1977 static int
1978 g_raid3_try_destroy(struct g_raid3_softc *sc)
1979 {
1980
1981         g_topology_assert_not();
1982         sx_assert(&sc->sc_lock, SX_XLOCKED);
1983
1984         if (sc->sc_rootmount != NULL) {
1985                 G_RAID3_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
1986                     sc->sc_rootmount);
1987                 root_mount_rel(sc->sc_rootmount);
1988                 sc->sc_rootmount = NULL;
1989         }
1990
1991         g_topology_lock();
1992         if (!g_raid3_can_destroy(sc)) {
1993                 g_topology_unlock();
1994                 return (0);
1995         }
1996         sc->sc_geom->softc = NULL;
1997         sc->sc_sync.ds_geom->softc = NULL;
1998         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_WAIT) != 0) {
1999                 g_topology_unlock();
2000                 G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__,
2001                     &sc->sc_worker);
2002                 /* Unlock sc_lock here, as it can be destroyed after wakeup. */
2003                 sx_xunlock(&sc->sc_lock);
2004                 wakeup(&sc->sc_worker);
2005                 sc->sc_worker = NULL;
2006         } else {
2007                 g_topology_unlock();
2008                 g_raid3_destroy_device(sc);
2009                 free(sc->sc_disks, M_RAID3);
2010                 free(sc, M_RAID3);
2011         }
2012         return (1);
2013 }
2014
2015 /*
2016  * Worker thread.
2017  */
2018 static void
2019 g_raid3_worker(void *arg)
2020 {
2021         struct g_raid3_softc *sc;
2022         struct g_raid3_event *ep;
2023         struct bio *bp;
2024         int timeout;
2025
2026         sc = arg;
2027         thread_lock(curthread);
2028         sched_prio(curthread, PRIBIO);
2029         thread_unlock(curthread);
2030
2031         sx_xlock(&sc->sc_lock);
2032         for (;;) {
2033                 G_RAID3_DEBUG(5, "%s: Let's see...", __func__);
2034                 /*
2035                  * First take a look at events.
2036                  * This is important to handle events before any I/O requests.
2037                  */
2038                 ep = g_raid3_event_get(sc);
2039                 if (ep != NULL) {
2040                         g_raid3_event_remove(sc, ep);
2041                         if ((ep->e_flags & G_RAID3_EVENT_DEVICE) != 0) {
2042                                 /* Update only device status. */
2043                                 G_RAID3_DEBUG(3,
2044                                     "Running event for device %s.",
2045                                     sc->sc_name);
2046                                 ep->e_error = 0;
2047                                 g_raid3_update_device(sc, 1);
2048                         } else {
2049                                 /* Update disk status. */
2050                                 G_RAID3_DEBUG(3, "Running event for disk %s.",
2051                                      g_raid3_get_diskname(ep->e_disk));
2052                                 ep->e_error = g_raid3_update_disk(ep->e_disk,
2053                                     ep->e_state);
2054                                 if (ep->e_error == 0)
2055                                         g_raid3_update_device(sc, 0);
2056                         }
2057                         if ((ep->e_flags & G_RAID3_EVENT_DONTWAIT) != 0) {
2058                                 KASSERT(ep->e_error == 0,
2059                                     ("Error cannot be handled."));
2060                                 g_raid3_event_free(ep);
2061                         } else {
2062                                 ep->e_flags |= G_RAID3_EVENT_DONE;
2063                                 G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__,
2064                                     ep);
2065                                 mtx_lock(&sc->sc_events_mtx);
2066                                 wakeup(ep);
2067                                 mtx_unlock(&sc->sc_events_mtx);
2068                         }
2069                         if ((sc->sc_flags &
2070                             G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
2071                                 if (g_raid3_try_destroy(sc)) {
2072                                         curthread->td_pflags &= ~TDP_GEOM;
2073                                         G_RAID3_DEBUG(1, "Thread exiting.");
2074                                         kproc_exit(0);
2075                                 }
2076                         }
2077                         G_RAID3_DEBUG(5, "%s: I'm here 1.", __func__);
2078                         continue;
2079                 }
2080                 /*
2081                  * Check if we can mark array as CLEAN and if we can't take
2082                  * how much seconds should we wait.
2083                  */
2084                 timeout = g_raid3_idle(sc, -1);
2085                 /*
2086                  * Now I/O requests.
2087                  */
2088                 /* Get first request from the queue. */
2089                 mtx_lock(&sc->sc_queue_mtx);
2090                 bp = bioq_first(&sc->sc_queue);
2091                 if (bp == NULL) {
2092                         if ((sc->sc_flags &
2093                             G_RAID3_DEVICE_FLAG_DESTROY) != 0) {
2094                                 mtx_unlock(&sc->sc_queue_mtx);
2095                                 if (g_raid3_try_destroy(sc)) {
2096                                         curthread->td_pflags &= ~TDP_GEOM;
2097                                         G_RAID3_DEBUG(1, "Thread exiting.");
2098                                         kproc_exit(0);
2099                                 }
2100                                 mtx_lock(&sc->sc_queue_mtx);
2101                         }
2102                         sx_xunlock(&sc->sc_lock);
2103                         /*
2104                          * XXX: We can miss an event here, because an event
2105                          *      can be added without sx-device-lock and without
2106                          *      mtx-queue-lock. Maybe I should just stop using
2107                          *      dedicated mutex for events synchronization and
2108                          *      stick with the queue lock?
2109                          *      The event will hang here until next I/O request
2110                          *      or next event is received.
2111                          */
2112                         MSLEEP(sc, &sc->sc_queue_mtx, PRIBIO | PDROP, "r3:w1",
2113                             timeout * hz);
2114                         sx_xlock(&sc->sc_lock);
2115                         G_RAID3_DEBUG(5, "%s: I'm here 4.", __func__);
2116                         continue;
2117                 }
2118 process:
2119                 bioq_remove(&sc->sc_queue, bp);
2120                 mtx_unlock(&sc->sc_queue_mtx);
2121
2122                 if (bp->bio_from->geom == sc->sc_sync.ds_geom &&
2123                     (bp->bio_cflags & G_RAID3_BIO_CFLAG_SYNC) != 0) {
2124                         g_raid3_sync_request(bp);       /* READ */
2125                 } else if (bp->bio_to != sc->sc_provider) {
2126                         if ((bp->bio_cflags & G_RAID3_BIO_CFLAG_REGULAR) != 0)
2127                                 g_raid3_regular_request(bp);
2128                         else if ((bp->bio_cflags & G_RAID3_BIO_CFLAG_SYNC) != 0)
2129                                 g_raid3_sync_request(bp);       /* WRITE */
2130                         else {
2131                                 KASSERT(0,
2132                                     ("Invalid request cflags=0x%hx to=%s.",
2133                                     bp->bio_cflags, bp->bio_to->name));
2134                         }
2135                 } else if (g_raid3_register_request(bp) != 0) {
2136                         mtx_lock(&sc->sc_queue_mtx);
2137                         bioq_insert_head(&sc->sc_queue, bp);
2138                         /*
2139                          * We are short in memory, let see if there are finished
2140                          * request we can free.
2141                          */
2142                         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
2143                                 if (bp->bio_cflags & G_RAID3_BIO_CFLAG_REGULAR)
2144                                         goto process;
2145                         }
2146                         /*
2147                          * No finished regular request, so at least keep
2148                          * synchronization running.
2149                          */
2150                         TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
2151                                 if (bp->bio_cflags & G_RAID3_BIO_CFLAG_SYNC)
2152                                         goto process;
2153                         }
2154                         sx_xunlock(&sc->sc_lock);
2155                         MSLEEP(&sc->sc_queue, &sc->sc_queue_mtx, PRIBIO | PDROP,
2156                             "r3:lowmem", hz / 10);
2157                         sx_xlock(&sc->sc_lock);
2158                 }
2159                 G_RAID3_DEBUG(5, "%s: I'm here 9.", __func__);
2160         }
2161 }
2162
2163 static void
2164 g_raid3_update_idle(struct g_raid3_softc *sc, struct g_raid3_disk *disk)
2165 {
2166
2167         sx_assert(&sc->sc_lock, SX_LOCKED);
2168         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) != 0)
2169                 return;
2170         if (!sc->sc_idle && (disk->d_flags & G_RAID3_DISK_FLAG_DIRTY) == 0) {
2171                 G_RAID3_DEBUG(1, "Disk %s (device %s) marked as dirty.",
2172                     g_raid3_get_diskname(disk), sc->sc_name);
2173                 disk->d_flags |= G_RAID3_DISK_FLAG_DIRTY;
2174         } else if (sc->sc_idle &&
2175             (disk->d_flags & G_RAID3_DISK_FLAG_DIRTY) != 0) {
2176                 G_RAID3_DEBUG(1, "Disk %s (device %s) marked as clean.",
2177                     g_raid3_get_diskname(disk), sc->sc_name);
2178                 disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
2179         }
2180 }
2181
2182 static void
2183 g_raid3_sync_start(struct g_raid3_softc *sc)
2184 {
2185         struct g_raid3_disk *disk;
2186         struct g_consumer *cp;
2187         struct bio *bp;
2188         int error;
2189         u_int n;
2190
2191         g_topology_assert_not();
2192         sx_assert(&sc->sc_lock, SX_XLOCKED);
2193
2194         KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED,
2195             ("Device not in DEGRADED state (%s, %u).", sc->sc_name,
2196             sc->sc_state));
2197         KASSERT(sc->sc_syncdisk == NULL, ("Syncdisk is not NULL (%s, %u).",
2198             sc->sc_name, sc->sc_state));
2199         disk = NULL;
2200         for (n = 0; n < sc->sc_ndisks; n++) {
2201                 if (sc->sc_disks[n].d_state != G_RAID3_DISK_STATE_SYNCHRONIZING)
2202                         continue;
2203                 disk = &sc->sc_disks[n];
2204                 break;
2205         }
2206         if (disk == NULL)
2207                 return;
2208
2209         sx_xunlock(&sc->sc_lock);
2210         g_topology_lock();
2211         cp = g_new_consumer(sc->sc_sync.ds_geom);
2212         error = g_attach(cp, sc->sc_provider);
2213         KASSERT(error == 0,
2214             ("Cannot attach to %s (error=%d).", sc->sc_name, error));
2215         error = g_access(cp, 1, 0, 0);
2216         KASSERT(error == 0, ("Cannot open %s (error=%d).", sc->sc_name, error));
2217         g_topology_unlock();
2218         sx_xlock(&sc->sc_lock);
2219
2220         G_RAID3_DEBUG(0, "Device %s: rebuilding provider %s.", sc->sc_name,
2221             g_raid3_get_diskname(disk));
2222         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOFAILSYNC) == 0)
2223                 disk->d_flags |= G_RAID3_DISK_FLAG_DIRTY;
2224         KASSERT(disk->d_sync.ds_consumer == NULL,
2225             ("Sync consumer already exists (device=%s, disk=%s).",
2226             sc->sc_name, g_raid3_get_diskname(disk)));
2227
2228         disk->d_sync.ds_consumer = cp;
2229         disk->d_sync.ds_consumer->private = disk;
2230         disk->d_sync.ds_consumer->index = 0;
2231         sc->sc_syncdisk = disk;
2232
2233         /*
2234          * Allocate memory for synchronization bios and initialize them.
2235          */
2236         disk->d_sync.ds_bios = malloc(sizeof(struct bio *) * g_raid3_syncreqs,
2237             M_RAID3, M_WAITOK);
2238         for (n = 0; n < g_raid3_syncreqs; n++) {
2239                 bp = g_alloc_bio();
2240                 disk->d_sync.ds_bios[n] = bp;
2241                 bp->bio_parent = NULL;
2242                 bp->bio_cmd = BIO_READ;
2243                 bp->bio_data = malloc(MAXPHYS, M_RAID3, M_WAITOK);
2244                 bp->bio_cflags = 0;
2245                 bp->bio_offset = disk->d_sync.ds_offset * (sc->sc_ndisks - 1);
2246                 bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset);
2247                 disk->d_sync.ds_offset += bp->bio_length / (sc->sc_ndisks - 1);
2248                 bp->bio_done = g_raid3_sync_done;
2249                 bp->bio_from = disk->d_sync.ds_consumer;
2250                 bp->bio_to = sc->sc_provider;
2251                 bp->bio_caller1 = (void *)(uintptr_t)n;
2252         }
2253
2254         /* Set the number of in-flight synchronization requests. */
2255         disk->d_sync.ds_inflight = g_raid3_syncreqs;
2256
2257         /*
2258          * Fire off first synchronization requests.
2259          */
2260         for (n = 0; n < g_raid3_syncreqs; n++) {
2261                 bp = disk->d_sync.ds_bios[n];
2262                 G_RAID3_LOGREQ(3, bp, "Sending synchronization request.");
2263                 disk->d_sync.ds_consumer->index++;
2264                 /*
2265                  * Delay the request if it is colliding with a regular request.
2266                  */
2267                 if (g_raid3_regular_collision(sc, bp))
2268                         g_raid3_sync_delay(sc, bp);
2269                 else
2270                         g_io_request(bp, disk->d_sync.ds_consumer);
2271         }
2272 }
2273
2274 /*
2275  * Stop synchronization process.
2276  * type: 0 - synchronization finished
2277  *       1 - synchronization stopped
2278  */
2279 static void
2280 g_raid3_sync_stop(struct g_raid3_softc *sc, int type)
2281 {
2282         struct g_raid3_disk *disk;
2283         struct g_consumer *cp;
2284
2285         g_topology_assert_not();
2286         sx_assert(&sc->sc_lock, SX_LOCKED);
2287
2288         KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED,
2289             ("Device not in DEGRADED state (%s, %u).", sc->sc_name,
2290             sc->sc_state));
2291         disk = sc->sc_syncdisk;
2292         sc->sc_syncdisk = NULL;
2293         KASSERT(disk != NULL, ("No disk was synchronized (%s).", sc->sc_name));
2294         KASSERT(disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING,
2295             ("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
2296             g_raid3_disk_state2str(disk->d_state)));
2297         if (disk->d_sync.ds_consumer == NULL)
2298                 return;
2299
2300         if (type == 0) {
2301                 G_RAID3_DEBUG(0, "Device %s: rebuilding provider %s finished.",
2302                     sc->sc_name, g_raid3_get_diskname(disk));
2303         } else /* if (type == 1) */ {
2304                 G_RAID3_DEBUG(0, "Device %s: rebuilding provider %s stopped.",
2305                     sc->sc_name, g_raid3_get_diskname(disk));
2306         }
2307         free(disk->d_sync.ds_bios, M_RAID3);
2308         disk->d_sync.ds_bios = NULL;
2309         cp = disk->d_sync.ds_consumer;
2310         disk->d_sync.ds_consumer = NULL;
2311         disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
2312         sx_xunlock(&sc->sc_lock); /* Avoid recursion on sc_lock. */
2313         g_topology_lock();
2314         g_raid3_kill_consumer(sc, cp);
2315         g_topology_unlock();
2316         sx_xlock(&sc->sc_lock);
2317 }
2318
2319 static void
2320 g_raid3_launch_provider(struct g_raid3_softc *sc)
2321 {
2322         struct g_provider *pp;
2323         struct g_raid3_disk *disk;
2324         int n;
2325
2326         sx_assert(&sc->sc_lock, SX_LOCKED);
2327
2328         g_topology_lock();
2329         pp = g_new_providerf(sc->sc_geom, "raid3/%s", sc->sc_name);
2330         pp->mediasize = sc->sc_mediasize;
2331         pp->sectorsize = sc->sc_sectorsize;
2332         pp->stripesize = 0;
2333         pp->stripeoffset = 0;
2334         for (n = 0; n < sc->sc_ndisks; n++) {
2335                 disk = &sc->sc_disks[n];
2336                 if (disk->d_consumer && disk->d_consumer->provider &&
2337                     disk->d_consumer->provider->stripesize > pp->stripesize) {
2338                         pp->stripesize = disk->d_consumer->provider->stripesize;
2339                         pp->stripeoffset = disk->d_consumer->provider->stripeoffset;
2340                 }
2341         }
2342         pp->stripesize *= sc->sc_ndisks - 1;
2343         pp->stripeoffset *= sc->sc_ndisks - 1;
2344         sc->sc_provider = pp;
2345         g_error_provider(pp, 0);
2346         g_topology_unlock();
2347         G_RAID3_DEBUG(0, "Device %s launched (%u/%u).", pp->name,
2348             g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE), sc->sc_ndisks);
2349
2350         if (sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED)
2351                 g_raid3_sync_start(sc);
2352 }
2353
2354 static void
2355 g_raid3_destroy_provider(struct g_raid3_softc *sc)
2356 {
2357         struct bio *bp;
2358
2359         g_topology_assert_not();
2360         KASSERT(sc->sc_provider != NULL, ("NULL provider (device=%s).",
2361             sc->sc_name));
2362
2363         g_topology_lock();
2364         g_error_provider(sc->sc_provider, ENXIO);
2365         mtx_lock(&sc->sc_queue_mtx);
2366         while ((bp = bioq_first(&sc->sc_queue)) != NULL) {
2367                 bioq_remove(&sc->sc_queue, bp);
2368                 g_io_deliver(bp, ENXIO);
2369         }
2370         mtx_unlock(&sc->sc_queue_mtx);
2371         G_RAID3_DEBUG(0, "Device %s: provider %s destroyed.", sc->sc_name,
2372             sc->sc_provider->name);
2373         g_wither_provider(sc->sc_provider, ENXIO);
2374         g_topology_unlock();
2375         sc->sc_provider = NULL;
2376         if (sc->sc_syncdisk != NULL)
2377                 g_raid3_sync_stop(sc, 1);
2378 }
2379
2380 static void
2381 g_raid3_go(void *arg)
2382 {
2383         struct g_raid3_softc *sc;
2384
2385         sc = arg;
2386         G_RAID3_DEBUG(0, "Force device %s start due to timeout.", sc->sc_name);
2387         g_raid3_event_send(sc, 0,
2388             G_RAID3_EVENT_DONTWAIT | G_RAID3_EVENT_DEVICE);
2389 }
2390
2391 static u_int
2392 g_raid3_determine_state(struct g_raid3_disk *disk)
2393 {
2394         struct g_raid3_softc *sc;
2395         u_int state;
2396
2397         sc = disk->d_softc;
2398         if (sc->sc_syncid == disk->d_sync.ds_syncid) {
2399                 if ((disk->d_flags &
2400                     G_RAID3_DISK_FLAG_SYNCHRONIZING) == 0) {
2401                         /* Disk does not need synchronization. */
2402                         state = G_RAID3_DISK_STATE_ACTIVE;
2403                 } else {
2404                         if ((sc->sc_flags &
2405                              G_RAID3_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2406                             (disk->d_flags &
2407                              G_RAID3_DISK_FLAG_FORCE_SYNC) != 0) {
2408                                 /*
2409                                  * We can start synchronization from
2410                                  * the stored offset.
2411                                  */
2412                                 state = G_RAID3_DISK_STATE_SYNCHRONIZING;
2413                         } else {
2414                                 state = G_RAID3_DISK_STATE_STALE;
2415                         }
2416                 }
2417         } else if (disk->d_sync.ds_syncid < sc->sc_syncid) {
2418                 /*
2419                  * Reset all synchronization data for this disk,
2420                  * because if it even was synchronized, it was
2421                  * synchronized to disks with different syncid.
2422                  */
2423                 disk->d_flags |= G_RAID3_DISK_FLAG_SYNCHRONIZING;
2424                 disk->d_sync.ds_offset = 0;
2425                 disk->d_sync.ds_offset_done = 0;
2426                 disk->d_sync.ds_syncid = sc->sc_syncid;
2427                 if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOAUTOSYNC) == 0 ||
2428                     (disk->d_flags & G_RAID3_DISK_FLAG_FORCE_SYNC) != 0) {
2429                         state = G_RAID3_DISK_STATE_SYNCHRONIZING;
2430                 } else {
2431                         state = G_RAID3_DISK_STATE_STALE;
2432                 }
2433         } else /* if (sc->sc_syncid < disk->d_sync.ds_syncid) */ {
2434                 /*
2435                  * Not good, NOT GOOD!
2436                  * It means that device was started on stale disks
2437                  * and more fresh disk just arrive.
2438                  * If there were writes, device is broken, sorry.
2439                  * I think the best choice here is don't touch
2440                  * this disk and inform the user loudly.
2441                  */
2442                 G_RAID3_DEBUG(0, "Device %s was started before the freshest "
2443                     "disk (%s) arrives!! It will not be connected to the "
2444                     "running device.", sc->sc_name,
2445                     g_raid3_get_diskname(disk));
2446                 g_raid3_destroy_disk(disk);
2447                 state = G_RAID3_DISK_STATE_NONE;
2448                 /* Return immediately, because disk was destroyed. */
2449                 return (state);
2450         }
2451         G_RAID3_DEBUG(3, "State for %s disk: %s.",
2452             g_raid3_get_diskname(disk), g_raid3_disk_state2str(state));
2453         return (state);
2454 }
2455
2456 /*
2457  * Update device state.
2458  */
2459 static void
2460 g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
2461 {
2462         struct g_raid3_disk *disk;
2463         u_int state;
2464
2465         sx_assert(&sc->sc_lock, SX_XLOCKED);
2466
2467         switch (sc->sc_state) {
2468         case G_RAID3_DEVICE_STATE_STARTING:
2469             {
2470                 u_int n, ndirty, ndisks, genid, syncid;
2471
2472                 KASSERT(sc->sc_provider == NULL,
2473                     ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
2474                 /*
2475                  * Are we ready? We are, if all disks are connected or
2476                  * one disk is missing and 'force' is true.
2477                  */
2478                 if (g_raid3_ndisks(sc, -1) + force == sc->sc_ndisks) {
2479                         if (!force)
2480                                 callout_drain(&sc->sc_callout);
2481                 } else {
2482                         if (force) {
2483                                 /*
2484                                  * Timeout expired, so destroy device.
2485                                  */
2486                                 sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
2487                                 G_RAID3_DEBUG(1, "root_mount_rel[%u] %p",
2488                                     __LINE__, sc->sc_rootmount);
2489                                 root_mount_rel(sc->sc_rootmount);
2490                                 sc->sc_rootmount = NULL;
2491                         }
2492                         return;
2493                 }
2494
2495                 /*
2496                  * Find the biggest genid.
2497                  */
2498                 genid = 0;
2499                 for (n = 0; n < sc->sc_ndisks; n++) {
2500                         disk = &sc->sc_disks[n];
2501                         if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
2502                                 continue;
2503                         if (disk->d_genid > genid)
2504                                 genid = disk->d_genid;
2505                 }
2506                 sc->sc_genid = genid;
2507                 /*
2508                  * Remove all disks without the biggest genid.
2509                  */
2510                 for (n = 0; n < sc->sc_ndisks; n++) {
2511                         disk = &sc->sc_disks[n];
2512                         if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
2513                                 continue;
2514                         if (disk->d_genid < genid) {
2515                                 G_RAID3_DEBUG(0,
2516                                     "Component %s (device %s) broken, skipping.",
2517                                     g_raid3_get_diskname(disk), sc->sc_name);
2518                                 g_raid3_destroy_disk(disk);
2519                         }
2520                 }
2521
2522                 /*
2523                  * There must be at least 'sc->sc_ndisks - 1' components
2524                  * with the same syncid and without SYNCHRONIZING flag.
2525                  */
2526
2527                 /*
2528                  * Find the biggest syncid, number of valid components and
2529                  * number of dirty components.
2530                  */
2531                 ndirty = ndisks = syncid = 0;
2532                 for (n = 0; n < sc->sc_ndisks; n++) {
2533                         disk = &sc->sc_disks[n];
2534                         if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
2535                                 continue;
2536                         if ((disk->d_flags & G_RAID3_DISK_FLAG_DIRTY) != 0)
2537                                 ndirty++;
2538                         if (disk->d_sync.ds_syncid > syncid) {
2539                                 syncid = disk->d_sync.ds_syncid;
2540                                 ndisks = 0;
2541                         } else if (disk->d_sync.ds_syncid < syncid) {
2542                                 continue;
2543                         }
2544                         if ((disk->d_flags &
2545                             G_RAID3_DISK_FLAG_SYNCHRONIZING) != 0) {
2546                                 continue;
2547                         }
2548                         ndisks++;
2549                 }
2550                 /*
2551                  * Do we have enough valid components?
2552                  */
2553                 if (ndisks + 1 < sc->sc_ndisks) {
2554                         G_RAID3_DEBUG(0,
2555                             "Device %s is broken, too few valid components.",
2556                             sc->sc_name);
2557                         sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
2558                         return;
2559                 }
2560                 /*
2561                  * If there is one DIRTY component and all disks are present,
2562                  * mark it for synchronization. If there is more than one DIRTY
2563                  * component, mark parity component for synchronization.
2564                  */
2565                 if (ndisks == sc->sc_ndisks && ndirty == 1) {
2566                         for (n = 0; n < sc->sc_ndisks; n++) {
2567                                 disk = &sc->sc_disks[n];
2568                                 if ((disk->d_flags &
2569                                     G_RAID3_DISK_FLAG_DIRTY) == 0) {
2570                                         continue;
2571                                 }
2572                                 disk->d_flags |=
2573                                     G_RAID3_DISK_FLAG_SYNCHRONIZING;
2574                         }
2575                 } else if (ndisks == sc->sc_ndisks && ndirty > 1) {
2576                         disk = &sc->sc_disks[sc->sc_ndisks - 1];
2577                         disk->d_flags |= G_RAID3_DISK_FLAG_SYNCHRONIZING;
2578                 }
2579
2580                 sc->sc_syncid = syncid;
2581                 if (force) {
2582                         /* Remember to bump syncid on first write. */
2583                         sc->sc_bump_id |= G_RAID3_BUMP_SYNCID;
2584                 }
2585                 if (ndisks == sc->sc_ndisks)
2586                         state = G_RAID3_DEVICE_STATE_COMPLETE;
2587                 else /* if (ndisks == sc->sc_ndisks - 1) */
2588                         state = G_RAID3_DEVICE_STATE_DEGRADED;
2589                 G_RAID3_DEBUG(1, "Device %s state changed from %s to %s.",
2590                     sc->sc_name, g_raid3_device_state2str(sc->sc_state),
2591                     g_raid3_device_state2str(state));
2592                 sc->sc_state = state;
2593                 for (n = 0; n < sc->sc_ndisks; n++) {
2594                         disk = &sc->sc_disks[n];
2595                         if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
2596                                 continue;
2597                         state = g_raid3_determine_state(disk);
2598                         g_raid3_event_send(disk, state, G_RAID3_EVENT_DONTWAIT);
2599                         if (state == G_RAID3_DISK_STATE_STALE)
2600                                 sc->sc_bump_id |= G_RAID3_BUMP_SYNCID;
2601                 }
2602                 break;
2603             }
2604         case G_RAID3_DEVICE_STATE_DEGRADED:
2605                 /*
2606                  * Genid need to be bumped immediately, so do it here.
2607                  */
2608                 if ((sc->sc_bump_id & G_RAID3_BUMP_GENID) != 0) {
2609                         sc->sc_bump_id &= ~G_RAID3_BUMP_GENID;
2610                         g_raid3_bump_genid(sc);
2611                 }
2612
2613                 if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NEW) > 0)
2614                         return;
2615                 if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) <
2616                     sc->sc_ndisks - 1) {
2617                         if (sc->sc_provider != NULL)
2618                                 g_raid3_destroy_provider(sc);
2619                         sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
2620                         return;
2621                 }
2622                 if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) ==
2623                     sc->sc_ndisks) {
2624                         state = G_RAID3_DEVICE_STATE_COMPLETE;
2625                         G_RAID3_DEBUG(1,
2626                             "Device %s state changed from %s to %s.",
2627                             sc->sc_name, g_raid3_device_state2str(sc->sc_state),
2628                             g_raid3_device_state2str(state));
2629                         sc->sc_state = state;
2630                 }
2631                 if (sc->sc_provider == NULL)
2632                         g_raid3_launch_provider(sc);
2633                 if (sc->sc_rootmount != NULL) {
2634                         G_RAID3_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2635                             sc->sc_rootmount);
2636                         root_mount_rel(sc->sc_rootmount);
2637                         sc->sc_rootmount = NULL;
2638                 }
2639                 break;
2640         case G_RAID3_DEVICE_STATE_COMPLETE:
2641                 /*
2642                  * Genid need to be bumped immediately, so do it here.
2643                  */
2644                 if ((sc->sc_bump_id & G_RAID3_BUMP_GENID) != 0) {
2645                         sc->sc_bump_id &= ~G_RAID3_BUMP_GENID;
2646                         g_raid3_bump_genid(sc);
2647                 }
2648
2649                 if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NEW) > 0)
2650                         return;
2651                 KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) >=
2652                     sc->sc_ndisks - 1,
2653                     ("Too few ACTIVE components in COMPLETE state (device %s).",
2654                     sc->sc_name));
2655                 if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) ==
2656                     sc->sc_ndisks - 1) {
2657                         state = G_RAID3_DEVICE_STATE_DEGRADED;
2658                         G_RAID3_DEBUG(1,
2659                             "Device %s state changed from %s to %s.",
2660                             sc->sc_name, g_raid3_device_state2str(sc->sc_state),
2661                             g_raid3_device_state2str(state));
2662                         sc->sc_state = state;
2663                 }
2664                 if (sc->sc_provider == NULL)
2665                         g_raid3_launch_provider(sc);
2666                 if (sc->sc_rootmount != NULL) {
2667                         G_RAID3_DEBUG(1, "root_mount_rel[%u] %p", __LINE__,
2668                             sc->sc_rootmount);
2669                         root_mount_rel(sc->sc_rootmount);
2670                         sc->sc_rootmount = NULL;
2671                 }
2672                 break;
2673         default:
2674                 KASSERT(1 == 0, ("Wrong device state (%s, %s).", sc->sc_name,
2675                     g_raid3_device_state2str(sc->sc_state)));
2676                 break;
2677         }
2678 }
2679
2680 /*
2681  * Update disk state and device state if needed.
2682  */
2683 #define DISK_STATE_CHANGED()    G_RAID3_DEBUG(1,                        \
2684         "Disk %s state changed from %s to %s (device %s).",             \
2685         g_raid3_get_diskname(disk),                                     \
2686         g_raid3_disk_state2str(disk->d_state),                          \
2687         g_raid3_disk_state2str(state), sc->sc_name)
2688 static int
2689 g_raid3_update_disk(struct g_raid3_disk *disk, u_int state)
2690 {
2691         struct g_raid3_softc *sc;
2692
2693         sc = disk->d_softc;
2694         sx_assert(&sc->sc_lock, SX_XLOCKED);
2695
2696 again:
2697         G_RAID3_DEBUG(3, "Changing disk %s state from %s to %s.",
2698             g_raid3_get_diskname(disk), g_raid3_disk_state2str(disk->d_state),
2699             g_raid3_disk_state2str(state));
2700         switch (state) {
2701         case G_RAID3_DISK_STATE_NEW:
2702                 /*
2703                  * Possible scenarios:
2704                  * 1. New disk arrive.
2705                  */
2706                 /* Previous state should be NONE. */
2707                 KASSERT(disk->d_state == G_RAID3_DISK_STATE_NONE,
2708                     ("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
2709                     g_raid3_disk_state2str(disk->d_state)));
2710                 DISK_STATE_CHANGED();
2711
2712                 disk->d_state = state;
2713                 G_RAID3_DEBUG(1, "Device %s: provider %s detected.",
2714                     sc->sc_name, g_raid3_get_diskname(disk));
2715                 if (sc->sc_state == G_RAID3_DEVICE_STATE_STARTING)
2716                         break;
2717                 KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
2718                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
2719                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2720                     g_raid3_device_state2str(sc->sc_state),
2721                     g_raid3_get_diskname(disk),
2722                     g_raid3_disk_state2str(disk->d_state)));
2723                 state = g_raid3_determine_state(disk);
2724                 if (state != G_RAID3_DISK_STATE_NONE)
2725                         goto again;
2726                 break;
2727         case G_RAID3_DISK_STATE_ACTIVE:
2728                 /*
2729                  * Possible scenarios:
2730                  * 1. New disk does not need synchronization.
2731                  * 2. Synchronization process finished successfully.
2732                  */
2733                 KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
2734                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
2735                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2736                     g_raid3_device_state2str(sc->sc_state),
2737                     g_raid3_get_diskname(disk),
2738                     g_raid3_disk_state2str(disk->d_state)));
2739                 /* Previous state should be NEW or SYNCHRONIZING. */
2740                 KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW ||
2741                     disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING,
2742                     ("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
2743                     g_raid3_disk_state2str(disk->d_state)));
2744                 DISK_STATE_CHANGED();
2745
2746                 if (disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
2747                         disk->d_flags &= ~G_RAID3_DISK_FLAG_SYNCHRONIZING;
2748                         disk->d_flags &= ~G_RAID3_DISK_FLAG_FORCE_SYNC;
2749                         g_raid3_sync_stop(sc, 0);
2750                 }
2751                 disk->d_state = state;
2752                 disk->d_sync.ds_offset = 0;
2753                 disk->d_sync.ds_offset_done = 0;
2754                 g_raid3_update_idle(sc, disk);
2755                 g_raid3_update_metadata(disk);
2756                 G_RAID3_DEBUG(1, "Device %s: provider %s activated.",
2757                     sc->sc_name, g_raid3_get_diskname(disk));
2758                 break;
2759         case G_RAID3_DISK_STATE_STALE:
2760                 /*
2761                  * Possible scenarios:
2762                  * 1. Stale disk was connected.
2763                  */
2764                 /* Previous state should be NEW. */
2765                 KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW,
2766                     ("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
2767                     g_raid3_disk_state2str(disk->d_state)));
2768                 KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
2769                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
2770                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2771                     g_raid3_device_state2str(sc->sc_state),
2772                     g_raid3_get_diskname(disk),
2773                     g_raid3_disk_state2str(disk->d_state)));
2774                 /*
2775                  * STALE state is only possible if device is marked
2776                  * NOAUTOSYNC.
2777                  */
2778                 KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_NOAUTOSYNC) != 0,
2779                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2780                     g_raid3_device_state2str(sc->sc_state),
2781                     g_raid3_get_diskname(disk),
2782                     g_raid3_disk_state2str(disk->d_state)));
2783                 DISK_STATE_CHANGED();
2784
2785                 disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
2786                 disk->d_state = state;
2787                 g_raid3_update_metadata(disk);
2788                 G_RAID3_DEBUG(0, "Device %s: provider %s is stale.",
2789                     sc->sc_name, g_raid3_get_diskname(disk));
2790                 break;
2791         case G_RAID3_DISK_STATE_SYNCHRONIZING:
2792                 /*
2793                  * Possible scenarios:
2794                  * 1. Disk which needs synchronization was connected.
2795                  */
2796                 /* Previous state should be NEW. */
2797                 KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW,
2798                     ("Wrong disk state (%s, %s).", g_raid3_get_diskname(disk),
2799                     g_raid3_disk_state2str(disk->d_state)));
2800                 KASSERT(sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
2801                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE,
2802                     ("Wrong device state (%s, %s, %s, %s).", sc->sc_name,
2803                     g_raid3_device_state2str(sc->sc_state),
2804                     g_raid3_get_diskname(disk),
2805                     g_raid3_disk_state2str(disk->d_state)));
2806                 DISK_STATE_CHANGED();
2807
2808                 if (disk->d_state == G_RAID3_DISK_STATE_NEW)
2809                         disk->d_flags &= ~G_RAID3_DISK_FLAG_DIRTY;
2810                 disk->d_state = state;
2811                 if (sc->sc_provider != NULL) {
2812                         g_raid3_sync_start(sc);
2813                         g_raid3_update_metadata(disk);
2814                 }
2815                 break;
2816         case G_RAID3_DISK_STATE_DISCONNECTED:
2817                 /*
2818                  * Possible scenarios:
2819                  * 1. Device wasn't running yet, but disk disappear.
2820                  * 2. Disk was active and disapppear.
2821                  * 3. Disk disappear during synchronization process.
2822                  */
2823                 if (sc->sc_state == G_RAID3_DEVICE_STATE_DEGRADED ||
2824                     sc->sc_state == G_RAID3_DEVICE_STATE_COMPLETE) {
2825                         /*
2826                          * Previous state should be ACTIVE, STALE or
2827                          * SYNCHRONIZING.
2828                          */
2829                         KASSERT(disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
2830                             disk->d_state == G_RAID3_DISK_STATE_STALE ||
2831                             disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING,
2832                             ("Wrong disk state (%s, %s).",
2833                             g_raid3_get_diskname(disk),
2834                             g_raid3_disk_state2str(disk->d_state)));
2835                 } else if (sc->sc_state == G_RAID3_DEVICE_STATE_STARTING) {
2836                         /* Previous state should be NEW. */
2837                         KASSERT(disk->d_state == G_RAID3_DISK_STATE_NEW,
2838                             ("Wrong disk state (%s, %s).",
2839                             g_raid3_get_diskname(disk),
2840                             g_raid3_disk_state2str(disk->d_state)));
2841                         /*
2842                          * Reset bumping syncid if disk disappeared in STARTING
2843                          * state.
2844                          */
2845                         if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID) != 0)
2846                                 sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
2847 #ifdef  INVARIANTS
2848                 } else {
2849                         KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
2850                             sc->sc_name,
2851                             g_raid3_device_state2str(sc->sc_state),
2852                             g_raid3_get_diskname(disk),
2853                             g_raid3_disk_state2str(disk->d_state)));
2854 #endif
2855                 }
2856                 DISK_STATE_CHANGED();
2857                 G_RAID3_DEBUG(0, "Device %s: provider %s disconnected.",
2858                     sc->sc_name, g_raid3_get_diskname(disk));
2859
2860                 g_raid3_destroy_disk(disk);
2861                 break;
2862         default:
2863                 KASSERT(1 == 0, ("Unknown state (%u).", state));
2864                 break;
2865         }
2866         return (0);
2867 }
2868 #undef  DISK_STATE_CHANGED
2869
2870 int
2871 g_raid3_read_metadata(struct g_consumer *cp, struct g_raid3_metadata *md)
2872 {
2873         struct g_provider *pp;
2874         u_char *buf;
2875         int error;
2876
2877         g_topology_assert();
2878
2879         error = g_access(cp, 1, 0, 0);
2880         if (error != 0)
2881                 return (error);
2882         pp = cp->provider;
2883         g_topology_unlock();
2884         /* Metadata are stored on last sector. */
2885         buf = g_read_data(cp, pp->mediasize - pp->sectorsize, pp->sectorsize,
2886             &error);
2887         g_topology_lock();
2888         g_access(cp, -1, 0, 0);
2889         if (buf == NULL) {
2890                 G_RAID3_DEBUG(1, "Cannot read metadata from %s (error=%d).",
2891                     cp->provider->name, error);
2892                 return (error);
2893         }
2894
2895         /* Decode metadata. */
2896         error = raid3_metadata_decode(buf, md);
2897         g_free(buf);
2898         if (strcmp(md->md_magic, G_RAID3_MAGIC) != 0)
2899                 return (EINVAL);
2900         if (md->md_version > G_RAID3_VERSION) {
2901                 G_RAID3_DEBUG(0,
2902                     "Kernel module is too old to handle metadata from %s.",
2903                     cp->provider->name);
2904                 return (EINVAL);
2905         }
2906         if (error != 0) {
2907                 G_RAID3_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
2908                     cp->provider->name);
2909                 return (error);
2910         }
2911         if (md->md_sectorsize > MAXPHYS) {
2912                 G_RAID3_DEBUG(0, "The blocksize is too big.");
2913                 return (EINVAL);
2914         }
2915
2916         return (0);
2917 }
2918
2919 static int
2920 g_raid3_check_metadata(struct g_raid3_softc *sc, struct g_provider *pp,
2921     struct g_raid3_metadata *md)
2922 {
2923
2924         if (md->md_no >= sc->sc_ndisks) {
2925                 G_RAID3_DEBUG(1, "Invalid disk %s number (no=%u), skipping.",
2926                     pp->name, md->md_no);
2927                 return (EINVAL);
2928         }
2929         if (sc->sc_disks[md->md_no].d_state != G_RAID3_DISK_STATE_NODISK) {
2930                 G_RAID3_DEBUG(1, "Disk %s (no=%u) already exists, skipping.",
2931                     pp->name, md->md_no);
2932                 return (EEXIST);
2933         }
2934         if (md->md_all != sc->sc_ndisks) {
2935                 G_RAID3_DEBUG(1,
2936                     "Invalid '%s' field on disk %s (device %s), skipping.",
2937                     "md_all", pp->name, sc->sc_name);
2938                 return (EINVAL);
2939         }
2940         if ((md->md_mediasize % md->md_sectorsize) != 0) {
2941                 G_RAID3_DEBUG(1, "Invalid metadata (mediasize %% sectorsize != "
2942                     "0) on disk %s (device %s), skipping.", pp->name,
2943                     sc->sc_name);
2944                 return (EINVAL);
2945         }
2946         if (md->md_mediasize != sc->sc_mediasize) {
2947                 G_RAID3_DEBUG(1,
2948                     "Invalid '%s' field on disk %s (device %s), skipping.",
2949                     "md_mediasize", pp->name, sc->sc_name);
2950                 return (EINVAL);
2951         }
2952         if ((md->md_mediasize % (sc->sc_ndisks - 1)) != 0) {
2953                 G_RAID3_DEBUG(1,
2954                     "Invalid '%s' field on disk %s (device %s), skipping.",
2955                     "md_mediasize", pp->name, sc->sc_name);
2956                 return (EINVAL);
2957         }
2958         if ((sc->sc_mediasize / (sc->sc_ndisks - 1)) > pp->mediasize) {
2959                 G_RAID3_DEBUG(1,
2960                     "Invalid size of disk %s (device %s), skipping.", pp->name,
2961                     sc->sc_name);
2962                 return (EINVAL);
2963         }
2964         if ((md->md_sectorsize / pp->sectorsize) < sc->sc_ndisks - 1) {
2965                 G_RAID3_DEBUG(1,
2966                     "Invalid '%s' field on disk %s (device %s), skipping.",
2967                     "md_sectorsize", pp->name, sc->sc_name);
2968                 return (EINVAL);
2969         }
2970         if (md->md_sectorsize != sc->sc_sectorsize) {
2971                 G_RAID3_DEBUG(1,
2972                     "Invalid '%s' field on disk %s (device %s), skipping.",
2973                     "md_sectorsize", pp->name, sc->sc_name);
2974                 return (EINVAL);
2975         }
2976         if ((sc->sc_sectorsize % pp->sectorsize) != 0) {
2977                 G_RAID3_DEBUG(1,
2978                     "Invalid sector size of disk %s (device %s), skipping.",
2979                     pp->name, sc->sc_name);
2980                 return (EINVAL);
2981         }
2982         if ((md->md_mflags & ~G_RAID3_DEVICE_FLAG_MASK) != 0) {
2983                 G_RAID3_DEBUG(1,
2984                     "Invalid device flags on disk %s (device %s), skipping.",
2985                     pp->name, sc->sc_name);
2986                 return (EINVAL);
2987         }
2988         if ((md->md_mflags & G_RAID3_DEVICE_FLAG_VERIFY) != 0 &&
2989             (md->md_mflags & G_RAID3_DEVICE_FLAG_ROUND_ROBIN) != 0) {
2990                 /*
2991                  * VERIFY and ROUND-ROBIN options are mutally exclusive.
2992                  */
2993                 G_RAID3_DEBUG(1, "Both VERIFY and ROUND-ROBIN flags exist on "
2994                     "disk %s (device %s), skipping.", pp->name, sc->sc_name);
2995                 return (EINVAL);
2996         }
2997         if ((md->md_dflags & ~G_RAID3_DISK_FLAG_MASK) != 0) {
2998                 G_RAID3_DEBUG(1,
2999                     "Invalid disk flags on disk %s (device %s), skipping.",
3000                     pp->name, sc->sc_name);
3001                 return (EINVAL);
3002         }
3003         return (0);
3004 }
3005
3006 int
3007 g_raid3_add_disk(struct g_raid3_softc *sc, struct g_provider *pp,
3008     struct g_raid3_metadata *md)
3009 {
3010         struct g_raid3_disk *disk;
3011         int error;
3012
3013         g_topology_assert_not();
3014         G_RAID3_DEBUG(2, "Adding disk %s.", pp->name);
3015
3016         error = g_raid3_check_metadata(sc, pp, md);
3017         if (error != 0)
3018                 return (error);
3019         if (sc->sc_state != G_RAID3_DEVICE_STATE_STARTING &&
3020             md->md_genid < sc->sc_genid) {
3021                 G_RAID3_DEBUG(0, "Component %s (device %s) broken, skipping.",
3022                     pp->name, sc->sc_name);
3023                 return (EINVAL);
3024         }
3025         disk = g_raid3_init_disk(sc, pp, md, &error);
3026         if (disk == NULL)
3027                 return (error);
3028         error = g_raid3_event_send(disk, G_RAID3_DISK_STATE_NEW,
3029             G_RAID3_EVENT_WAIT);
3030         if (error != 0)
3031                 return (error);
3032         if (md->md_version < G_RAID3_VERSION) {
3033                 G_RAID3_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
3034                     pp->name, md->md_version, G_RAID3_VERSION);
3035                 g_raid3_update_metadata(disk);
3036         }
3037         return (0);
3038 }
3039
3040 static void
3041 g_raid3_destroy_delayed(void *arg, int flag)
3042 {
3043         struct g_raid3_softc *sc;
3044         int error;
3045
3046         if (flag == EV_CANCEL) {
3047                 G_RAID3_DEBUG(1, "Destroying canceled.");
3048                 return;
3049         }
3050         sc = arg;
3051         g_topology_unlock();
3052         sx_xlock(&sc->sc_lock);
3053         KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) == 0,
3054             ("DESTROY flag set on %s.", sc->sc_name));
3055         KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0,
3056             ("DESTROYING flag not set on %s.", sc->sc_name));
3057         G_RAID3_DEBUG(0, "Destroying %s (delayed).", sc->sc_name);
3058         error = g_raid3_destroy(sc, G_RAID3_DESTROY_SOFT);
3059         if (error != 0) {
3060                 G_RAID3_DEBUG(0, "Cannot destroy %s.", sc->sc_name);
3061                 sx_xunlock(&sc->sc_lock);
3062         }
3063         g_topology_lock();
3064 }
3065
3066 static int
3067 g_raid3_access(struct g_provider *pp, int acr, int acw, int ace)
3068 {
3069         struct g_raid3_softc *sc;
3070         int dcr, dcw, dce, error = 0;
3071
3072         g_topology_assert();
3073         G_RAID3_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
3074             acw, ace);
3075
3076         sc = pp->geom->softc;
3077         if (sc == NULL && acr <= 0 && acw <= 0 && ace <= 0)
3078                 return (0);
3079         KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
3080
3081         dcr = pp->acr + acr;
3082         dcw = pp->acw + acw;
3083         dce = pp->ace + ace;
3084
3085         g_topology_unlock();
3086         sx_xlock(&sc->sc_lock);
3087         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0 ||
3088             g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) < sc->sc_ndisks - 1) {
3089                 if (acr > 0 || acw > 0 || ace > 0)
3090                         error = ENXIO;
3091                 goto end;
3092         }
3093         if (dcw == 0)
3094                 g_raid3_idle(sc, dcw);
3095         if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0) {
3096                 if (acr > 0 || acw > 0 || ace > 0) {
3097                         error = ENXIO;
3098                         goto end;
3099                 }
3100                 if (dcr == 0 && dcw == 0 && dce == 0) {
3101                         g_post_event(g_raid3_destroy_delayed, sc, M_WAITOK,
3102                             sc, NULL);
3103                 }
3104         }
3105 end:
3106         sx_xunlock(&sc->sc_lock);
3107         g_topology_lock();
3108         return (error);
3109 }
3110
3111 static struct g_geom *
3112 g_raid3_create(struct g_class *mp, const struct g_raid3_metadata *md)
3113 {
3114         struct g_raid3_softc *sc;
3115         struct g_geom *gp;
3116         int error, timeout;
3117         u_int n;
3118
3119         g_topology_assert();
3120         G_RAID3_DEBUG(1, "Creating device %s (id=%u).", md->md_name, md->md_id);
3121
3122         /* One disk is minimum. */
3123         if (md->md_all < 1)
3124                 return (NULL);
3125         /*
3126          * Action geom.
3127          */
3128         gp = g_new_geomf(mp, "%s", md->md_name);
3129         sc = malloc(sizeof(*sc), M_RAID3, M_WAITOK | M_ZERO);
3130         sc->sc_disks = malloc(sizeof(struct g_raid3_disk) * md->md_all, M_RAID3,
3131             M_WAITOK | M_ZERO);
3132         gp->start = g_raid3_start;
3133         gp->orphan = g_raid3_orphan;
3134         gp->access = g_raid3_access;
3135         gp->dumpconf = g_raid3_dumpconf;
3136
3137         sc->sc_id = md->md_id;
3138         sc->sc_mediasize = md->md_mediasize;
3139         sc->sc_sectorsize = md->md_sectorsize;
3140         sc->sc_ndisks = md->md_all;
3141         sc->sc_round_robin = 0;
3142         sc->sc_flags = md->md_mflags;
3143         sc->sc_bump_id = 0;
3144         sc->sc_idle = 1;
3145         sc->sc_last_write = time_uptime;
3146         sc->sc_writes = 0;
3147         for (n = 0; n < sc->sc_ndisks; n++) {
3148                 sc->sc_disks[n].d_softc = sc;
3149                 sc->sc_disks[n].d_no = n;
3150                 sc->sc_disks[n].d_state = G_RAID3_DISK_STATE_NODISK;
3151         }
3152         sx_init(&sc->sc_lock, "graid3:lock");
3153         bioq_init(&sc->sc_queue);
3154         mtx_init(&sc->sc_queue_mtx, "graid3:queue", NULL, MTX_DEF);
3155         bioq_init(&sc->sc_regular_delayed);
3156         bioq_init(&sc->sc_inflight);
3157         bioq_init(&sc->sc_sync_delayed);
3158         TAILQ_INIT(&sc->sc_events);
3159         mtx_init(&sc->sc_events_mtx, "graid3:events", NULL, MTX_DEF);
3160         callout_init(&sc->sc_callout, 1);
3161         sc->sc_state = G_RAID3_DEVICE_STATE_STARTING;
3162         gp->softc = sc;
3163         sc->sc_geom = gp;
3164         sc->sc_provider = NULL;
3165         /*
3166          * Synchronization geom.
3167          */
3168         gp = g_new_geomf(mp, "%s.sync", md->md_name);
3169         gp->softc = sc;
3170         gp->orphan = g_raid3_orphan;
3171         sc->sc_sync.ds_geom = gp;
3172
3173         if (!g_raid3_use_malloc) {
3174                 sc->sc_zones[G_RAID3_ZONE_64K].sz_zone = uma_zcreate("gr3:64k",
3175                     65536, g_raid3_uma_ctor, g_raid3_uma_dtor, NULL, NULL,
3176                     UMA_ALIGN_PTR, 0);
3177                 sc->sc_zones[G_RAID3_ZONE_64K].sz_inuse = 0;
3178                 sc->sc_zones[G_RAID3_ZONE_64K].sz_max = g_raid3_n64k;
3179                 sc->sc_zones[G_RAID3_ZONE_64K].sz_requested =
3180                     sc->sc_zones[G_RAID3_ZONE_64K].sz_failed = 0;
3181                 sc->sc_zones[G_RAID3_ZONE_16K].sz_zone = uma_zcreate("gr3:16k",
3182                     16384, g_raid3_uma_ctor, g_raid3_uma_dtor, NULL, NULL,
3183                     UMA_ALIGN_PTR, 0);
3184                 sc->sc_zones[G_RAID3_ZONE_16K].sz_inuse = 0;
3185                 sc->sc_zones[G_RAID3_ZONE_16K].sz_max = g_raid3_n16k;
3186                 sc->sc_zones[G_RAID3_ZONE_16K].sz_requested =
3187                     sc->sc_zones[G_RAID3_ZONE_16K].sz_failed = 0;
3188                 sc->sc_zones[G_RAID3_ZONE_4K].sz_zone = uma_zcreate("gr3:4k",
3189                     4096, g_raid3_uma_ctor, g_raid3_uma_dtor, NULL, NULL,
3190                     UMA_ALIGN_PTR, 0);
3191                 sc->sc_zones[G_RAID3_ZONE_4K].sz_inuse = 0;
3192                 sc->sc_zones[G_RAID3_ZONE_4K].sz_max = g_raid3_n4k;
3193                 sc->sc_zones[G_RAID3_ZONE_4K].sz_requested =
3194                     sc->sc_zones[G_RAID3_ZONE_4K].sz_failed = 0;
3195         }
3196
3197         error = kproc_create(g_raid3_worker, sc, &sc->sc_worker, 0, 0,
3198             "g_raid3 %s", md->md_name);
3199         if (error != 0) {
3200                 G_RAID3_DEBUG(1, "Cannot create kernel thread for %s.",
3201                     sc->sc_name);
3202                 if (!g_raid3_use_malloc) {
3203                         uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_64K].sz_zone);
3204                         uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_16K].sz_zone);
3205                         uma_zdestroy(sc->sc_zones[G_RAID3_ZONE_4K].sz_zone);
3206                 }
3207                 g_destroy_geom(sc->sc_sync.ds_geom);
3208                 mtx_destroy(&sc->sc_events_mtx);
3209                 mtx_destroy(&sc->sc_queue_mtx);
3210                 sx_destroy(&sc->sc_lock);
3211                 g_destroy_geom(sc->sc_geom);
3212                 free(sc->sc_disks, M_RAID3);
3213                 free(sc, M_RAID3);
3214                 return (NULL);
3215         }
3216
3217         G_RAID3_DEBUG(1, "Device %s created (%u components, id=%u).",
3218             sc->sc_name, sc->sc_ndisks, sc->sc_id);
3219
3220         sc->sc_rootmount = root_mount_hold("GRAID3");
3221         G_RAID3_DEBUG(1, "root_mount_hold %p", sc->sc_rootmount);
3222
3223         /*
3224          * Run timeout.
3225          */
3226         timeout = atomic_load_acq_int(&g_raid3_timeout);
3227         callout_reset(&sc->sc_callout, timeout * hz, g_raid3_go, sc);
3228         return (sc->sc_geom);
3229 }
3230
3231 int
3232 g_raid3_destroy(struct g_raid3_softc *sc, int how)
3233 {
3234         struct g_provider *pp;
3235
3236         g_topology_assert_not();
3237         if (sc == NULL)
3238                 return (ENXIO);
3239         sx_assert(&sc->sc_lock, SX_XLOCKED);
3240
3241         pp = sc->sc_provider;
3242         if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
3243                 switch (how) {
3244                 case G_RAID3_DESTROY_SOFT:
3245                         G_RAID3_DEBUG(1,
3246                             "Device %s is still open (r%dw%de%d).", pp->name,
3247                             pp->acr, pp->acw, pp->ace);
3248                         return (EBUSY);
3249                 case G_RAID3_DESTROY_DELAYED:
3250                         G_RAID3_DEBUG(1,
3251                             "Device %s will be destroyed on last close.",
3252                             pp->name);
3253                         if (sc->sc_syncdisk != NULL)
3254                                 g_raid3_sync_stop(sc, 1);
3255                         sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROYING;
3256                         return (EBUSY);
3257                 case G_RAID3_DESTROY_HARD:
3258                         G_RAID3_DEBUG(1, "Device %s is still open, so it "
3259                             "can't be definitely removed.", pp->name);
3260                         break;
3261                 }
3262         }
3263
3264         g_topology_lock();
3265         if (sc->sc_geom->softc == NULL) {
3266                 g_topology_unlock();
3267                 return (0);
3268         }
3269         sc->sc_geom->softc = NULL;
3270         sc->sc_sync.ds_geom->softc = NULL;
3271         g_topology_unlock();
3272
3273         sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROY;
3274         sc->sc_flags |= G_RAID3_DEVICE_FLAG_WAIT;
3275         G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__, sc);
3276         sx_xunlock(&sc->sc_lock);
3277         mtx_lock(&sc->sc_queue_mtx);
3278         wakeup(sc);
3279         wakeup(&sc->sc_queue);
3280         mtx_unlock(&sc->sc_queue_mtx);
3281         G_RAID3_DEBUG(4, "%s: Sleeping %p.", __func__, &sc->sc_worker);
3282         while (sc->sc_worker != NULL)
3283                 tsleep(&sc->sc_worker, PRIBIO, "r3:destroy", hz / 5);
3284         G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, &sc->sc_worker);
3285         sx_xlock(&sc->sc_lock);
3286         g_raid3_destroy_device(sc);
3287         free(sc->sc_disks, M_RAID3);
3288         free(sc, M_RAID3);
3289         return (0);
3290 }
3291
3292 static void
3293 g_raid3_taste_orphan(struct g_consumer *cp)
3294 {
3295
3296         KASSERT(1 == 0, ("%s called while tasting %s.", __func__,
3297             cp->provider->name));
3298 }
3299
3300 static struct g_geom *
3301 g_raid3_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
3302 {
3303         struct g_raid3_metadata md;
3304         struct g_raid3_softc *sc;
3305         struct g_consumer *cp;
3306         struct g_geom *gp;
3307         int error;
3308
3309         g_topology_assert();
3310         g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
3311         G_RAID3_DEBUG(2, "Tasting %s.", pp->name);
3312
3313         gp = g_new_geomf(mp, "raid3:taste");
3314         /* This orphan function should be never called. */
3315         gp->orphan = g_raid3_taste_orphan;
3316         cp = g_new_consumer(gp);
3317         g_attach(cp, pp);
3318         error = g_raid3_read_metadata(cp, &md);
3319         g_detach(cp);
3320         g_destroy_consumer(cp);
3321         g_destroy_geom(gp);
3322         if (error != 0)
3323                 return (NULL);
3324         gp = NULL;
3325
3326         if (md.md_provider[0] != '\0' &&
3327             !g_compare_names(md.md_provider, pp->name))
3328                 return (NULL);
3329         if (md.md_provsize != 0 && md.md_provsize != pp->mediasize)
3330                 return (NULL);
3331         if (g_raid3_debug >= 2)
3332                 raid3_metadata_dump(&md);
3333
3334         /*
3335          * Let's check if device already exists.
3336          */
3337         sc = NULL;
3338         LIST_FOREACH(gp, &mp->geom, geom) {
3339                 sc = gp->softc;
3340                 if (sc == NULL)
3341                         continue;
3342                 if (sc->sc_sync.ds_geom == gp)
3343                         continue;
3344                 if (strcmp(md.md_name, sc->sc_name) != 0)
3345                         continue;
3346                 if (md.md_id != sc->sc_id) {
3347                         G_RAID3_DEBUG(0, "Device %s already configured.",
3348                             sc->sc_name);
3349                         return (NULL);
3350                 }
3351                 break;
3352         }
3353         if (gp == NULL) {
3354                 gp = g_raid3_create(mp, &md);
3355                 if (gp == NULL) {
3356                         G_RAID3_DEBUG(0, "Cannot create device %s.",
3357                             md.md_name);
3358                         return (NULL);
3359                 }
3360                 sc = gp->softc;
3361         }
3362         G_RAID3_DEBUG(1, "Adding disk %s to %s.", pp->name, gp->name);
3363         g_topology_unlock();
3364         sx_xlock(&sc->sc_lock);
3365         error = g_raid3_add_disk(sc, pp, &md);
3366         if (error != 0) {
3367                 G_RAID3_DEBUG(0, "Cannot add disk %s to %s (error=%d).",
3368                     pp->name, gp->name, error);
3369                 if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NODISK) ==
3370                     sc->sc_ndisks) {
3371                         g_cancel_event(sc);
3372                         g_raid3_destroy(sc, G_RAID3_DESTROY_HARD);
3373                         g_topology_lock();
3374                         return (NULL);
3375                 }
3376                 gp = NULL;
3377         }
3378         sx_xunlock(&sc->sc_lock);
3379         g_topology_lock();
3380         return (gp);
3381 }
3382
3383 static int
3384 g_raid3_destroy_geom(struct gctl_req *req __unused, struct g_class *mp __unused,
3385     struct g_geom *gp)
3386 {
3387         struct g_raid3_softc *sc;
3388         int error;
3389
3390         g_topology_unlock();
3391         sc = gp->softc;
3392         sx_xlock(&sc->sc_lock);
3393         g_cancel_event(sc);
3394         error = g_raid3_destroy(gp->softc, G_RAID3_DESTROY_SOFT);
3395         if (error != 0)
3396                 sx_xunlock(&sc->sc_lock);
3397         g_topology_lock();
3398         return (error);
3399 }
3400
3401 static void
3402 g_raid3_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
3403     struct g_consumer *cp, struct g_provider *pp)
3404 {
3405         struct g_raid3_softc *sc;
3406
3407         g_topology_assert();
3408
3409         sc = gp->softc;
3410         if (sc == NULL)
3411                 return;
3412         /* Skip synchronization geom. */
3413         if (gp == sc->sc_sync.ds_geom)
3414                 return;
3415         if (pp != NULL) {
3416                 /* Nothing here. */
3417         } else if (cp != NULL) {
3418                 struct g_raid3_disk *disk;
3419
3420                 disk = cp->private;
3421                 if (disk == NULL)
3422                         return;
3423                 g_topology_unlock();
3424                 sx_xlock(&sc->sc_lock);
3425                 sbuf_printf(sb, "%s<Type>", indent);
3426                 if (disk->d_no == sc->sc_ndisks - 1)
3427                         sbuf_printf(sb, "PARITY");
3428                 else
3429                         sbuf_printf(sb, "DATA");
3430                 sbuf_printf(sb, "</Type>\n");
3431                 sbuf_printf(sb, "%s<Number>%u</Number>\n", indent,
3432                     (u_int)disk->d_no);
3433                 if (disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
3434                         sbuf_printf(sb, "%s<Synchronized>", indent);
3435                         if (disk->d_sync.ds_offset == 0)
3436                                 sbuf_printf(sb, "0%%");
3437                         else {
3438                                 sbuf_printf(sb, "%u%%",
3439                                     (u_int)((disk->d_sync.ds_offset * 100) /
3440                                     (sc->sc_mediasize / (sc->sc_ndisks - 1))));
3441                         }
3442                         sbuf_printf(sb, "</Synchronized>\n");
3443                         if (disk->d_sync.ds_offset > 0) {
3444                                 sbuf_printf(sb, "%s<BytesSynced>%jd"
3445                                     "</BytesSynced>\n", indent,
3446                                     (intmax_t)disk->d_sync.ds_offset);
3447                         }
3448                 }
3449                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
3450                     disk->d_sync.ds_syncid);
3451                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, disk->d_genid);
3452                 sbuf_printf(sb, "%s<Flags>", indent);
3453                 if (disk->d_flags == 0)
3454                         sbuf_printf(sb, "NONE");
3455                 else {
3456                         int first = 1;
3457
3458 #define ADD_FLAG(flag, name)    do {                                    \
3459         if ((disk->d_flags & (flag)) != 0) {                            \
3460                 if (!first)                                             \
3461                         sbuf_printf(sb, ", ");                          \
3462                 else                                                    \
3463                         first = 0;                                      \
3464                 sbuf_printf(sb, name);                                  \
3465         }                                                               \
3466 } while (0)
3467                         ADD_FLAG(G_RAID3_DISK_FLAG_DIRTY, "DIRTY");
3468                         ADD_FLAG(G_RAID3_DISK_FLAG_HARDCODED, "HARDCODED");
3469                         ADD_FLAG(G_RAID3_DISK_FLAG_SYNCHRONIZING,
3470                             "SYNCHRONIZING");
3471                         ADD_FLAG(G_RAID3_DISK_FLAG_FORCE_SYNC, "FORCE_SYNC");
3472                         ADD_FLAG(G_RAID3_DISK_FLAG_BROKEN, "BROKEN");
3473 #undef  ADD_FLAG
3474                 }
3475                 sbuf_printf(sb, "</Flags>\n");
3476                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3477                     g_raid3_disk_state2str(disk->d_state));
3478                 sx_xunlock(&sc->sc_lock);
3479                 g_topology_lock();
3480         } else {
3481                 g_topology_unlock();
3482                 sx_xlock(&sc->sc_lock);
3483                 if (!g_raid3_use_malloc) {
3484                         sbuf_printf(sb,
3485                             "%s<Zone4kRequested>%u</Zone4kRequested>\n", indent,
3486                             sc->sc_zones[G_RAID3_ZONE_4K].sz_requested);
3487                         sbuf_printf(sb,
3488                             "%s<Zone4kFailed>%u</Zone4kFailed>\n", indent,
3489                             sc->sc_zones[G_RAID3_ZONE_4K].sz_failed);
3490                         sbuf_printf(sb,
3491                             "%s<Zone16kRequested>%u</Zone16kRequested>\n", indent,
3492                             sc->sc_zones[G_RAID3_ZONE_16K].sz_requested);
3493                         sbuf_printf(sb,
3494                             "%s<Zone16kFailed>%u</Zone16kFailed>\n", indent,
3495                             sc->sc_zones[G_RAID3_ZONE_16K].sz_failed);
3496                         sbuf_printf(sb,
3497                             "%s<Zone64kRequested>%u</Zone64kRequested>\n", indent,
3498                             sc->sc_zones[G_RAID3_ZONE_64K].sz_requested);
3499                         sbuf_printf(sb,
3500                             "%s<Zone64kFailed>%u</Zone64kFailed>\n", indent,
3501                             sc->sc_zones[G_RAID3_ZONE_64K].sz_failed);
3502                 }
3503                 sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
3504                 sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
3505                 sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
3506                 sbuf_printf(sb, "%s<Flags>", indent);
3507                 if (sc->sc_flags == 0)
3508                         sbuf_printf(sb, "NONE");
3509                 else {
3510                         int first = 1;
3511
3512 #define ADD_FLAG(flag, name)    do {                                    \
3513         if ((sc->sc_flags & (flag)) != 0) {                             \
3514                 if (!first)                                             \
3515                         sbuf_printf(sb, ", ");                          \
3516                 else                                                    \
3517                         first = 0;                                      \
3518                 sbuf_printf(sb, name);                                  \
3519         }                                                               \
3520 } while (0)
3521                         ADD_FLAG(G_RAID3_DEVICE_FLAG_NOFAILSYNC, "NOFAILSYNC");
3522                         ADD_FLAG(G_RAID3_DEVICE_FLAG_NOAUTOSYNC, "NOAUTOSYNC");
3523                         ADD_FLAG(G_RAID3_DEVICE_FLAG_ROUND_ROBIN,
3524                             "ROUND-ROBIN");
3525                         ADD_FLAG(G_RAID3_DEVICE_FLAG_VERIFY, "VERIFY");
3526 #undef  ADD_FLAG
3527                 }
3528                 sbuf_printf(sb, "</Flags>\n");
3529                 sbuf_printf(sb, "%s<Components>%u</Components>\n", indent,
3530                     sc->sc_ndisks);
3531                 sbuf_printf(sb, "%s<State>%s</State>\n", indent,
3532                     g_raid3_device_state2str(sc->sc_state));
3533                 sx_xunlock(&sc->sc_lock);
3534                 g_topology_lock();
3535         }
3536 }
3537
3538 static void
3539 g_raid3_shutdown_post_sync(void *arg, int howto)
3540 {
3541         struct g_class *mp;
3542         struct g_geom *gp, *gp2;
3543         struct g_raid3_softc *sc;
3544         int error;
3545
3546         mp = arg;
3547         g_topology_lock();
3548         g_raid3_shutdown = 1;
3549         LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
3550                 if ((sc = gp->softc) == NULL)
3551                         continue;
3552                 /* Skip synchronization geom. */
3553                 if (gp == sc->sc_sync.ds_geom)
3554                         continue;
3555                 g_topology_unlock();
3556                 sx_xlock(&sc->sc_lock);
3557                 g_raid3_idle(sc, -1);
3558                 g_cancel_event(sc);
3559                 error = g_raid3_destroy(sc, G_RAID3_DESTROY_DELAYED);
3560                 if (error != 0)
3561                         sx_xunlock(&sc->sc_lock);
3562                 g_topology_lock();
3563         }
3564         g_topology_unlock();
3565 }
3566
3567 static void
3568 g_raid3_init(struct g_class *mp)
3569 {
3570
3571         g_raid3_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
3572             g_raid3_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
3573         if (g_raid3_post_sync == NULL)
3574                 G_RAID3_DEBUG(0, "Warning! Cannot register shutdown event.");
3575 }
3576
3577 static void
3578 g_raid3_fini(struct g_class *mp)
3579 {
3580
3581         if (g_raid3_post_sync != NULL)
3582                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_raid3_post_sync);
3583 }
3584
3585 DECLARE_GEOM_CLASS(g_raid3_class, g_raid3);
3586 MODULE_VERSION(geom_raid3, 0);