sys/geom/raid/md_promise.c

   1 /*-
   2  * Copyright (c) 2011 Alexander Motin <mav@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/bio.h>
  32 #include <sys/endian.h>
  33 #include <sys/kernel.h>
  34 #include <sys/kobj.h>
  35 #include <sys/limits.h>
  36 #include <sys/lock.h>
  37 #include <sys/malloc.h>
  38 #include <sys/mutex.h>
  39 #include <sys/systm.h>
  40 #include <geom/geom.h>
  41 #include "geom/raid/g_raid.h"
  42 #include "g_raid_md_if.h"
  43
  44 static MALLOC_DEFINE(M_MD_PROMISE, "md_promise_data", "GEOM_RAID Promise metadata");
  45
  46 #define PROMISE_MAX_DISKS       8
  47 #define PROMISE_MAX_SUBDISKS    2
  48 #define PROMISE_META_OFFSET     14
  49
  50 struct promise_raid_disk {
  51         uint8_t         flags;                  /* Subdisk status. */
  52 #define PROMISE_F_VALID         0x01
  53 #define PROMISE_F_ONLINE        0x02
  54 #define PROMISE_F_ASSIGNED      0x04
  55 #define PROMISE_F_SPARE         0x08
  56 #define PROMISE_F_DUPLICATE     0x10
  57 #define PROMISE_F_REDIR         0x20
  58 #define PROMISE_F_DOWN          0x40
  59 #define PROMISE_F_READY         0x80
  60
  61         uint8_t         number;                 /* Position in a volume. */
  62         uint8_t         channel;                /* ATA channel number. */
  63         uint8_t         device;                 /* ATA device number. */
  64         uint64_t        id __packed;            /* Subdisk ID. */
  65 } __packed;
  66
  67 struct promise_raid_conf {
  68         char            promise_id[24];
  69 #define PROMISE_MAGIC           "Promise Technology, Inc."
  70 #define FREEBSD_MAGIC           "FreeBSD ATA driver RAID "
  71
  72         uint32_t        dummy_0;
  73         uint64_t        magic_0;
  74 #define PROMISE_MAGIC0(x)       (((uint64_t)(x.channel) << 48) | \
  75                                 ((uint64_t)(x.device != 0) << 56))
  76         uint16_t        magic_1;
  77         uint32_t        magic_2;
  78         uint8_t         filler1[470];
  79
  80         uint32_t        integrity;
  81 #define PROMISE_I_VALID         0x00000080
  82
  83         struct promise_raid_disk        disk;   /* This subdisk info. */
  84         uint32_t        disk_offset;            /* Subdisk offset. */
  85         uint32_t        disk_sectors;           /* Subdisk size */
  86         uint32_t        rebuild_lba;            /* Rebuild position. */
  87         uint16_t        generation;             /* Generation number. */
  88         uint8_t         status;                 /* Volume status. */
  89 #define PROMISE_S_VALID         0x01
  90 #define PROMISE_S_ONLINE        0x02
  91 #define PROMISE_S_INITED        0x04
  92 #define PROMISE_S_READY         0x08
  93 #define PROMISE_S_DEGRADED      0x10
  94 #define PROMISE_S_MARKED        0x20
  95 #define PROMISE_S_MIGRATING     0x40
  96 #define PROMISE_S_FUNCTIONAL    0x80
  97
  98         uint8_t         type;                   /* Voluem type. */
  99 #define PROMISE_T_RAID0         0x00
 100 #define PROMISE_T_RAID1         0x01
 101 #define PROMISE_T_RAID3         0x02
 102 #define PROMISE_T_RAID5         0x04
 103 #define PROMISE_T_SPAN          0x08
 104 #define PROMISE_T_JBOD          0x10
 105
 106         uint8_t         total_disks;            /* Disks in this volume. */
 107         uint8_t         stripe_shift;           /* Strip size. */
 108         uint8_t         array_width;            /* Number of RAID0 stripes. */
 109         uint8_t         array_number;           /* Global volume number. */
 110         uint32_t        total_sectors;          /* Volume size. */
 111         uint16_t        cylinders;              /* Volume geometry: C. */
 112         uint8_t         heads;                  /* Volume geometry: H. */
 113         uint8_t         sectors;                /* Volume geometry: S. */
 114         uint64_t        volume_id __packed;     /* Volume ID, */
 115         struct promise_raid_disk        disks[PROMISE_MAX_DISKS];
 116                                                 /* Subdisks in this volume. */
 117         char            name[32];               /* Volume label. */
 118
 119         uint32_t        filler2[8];
 120         uint32_t        magic_3;        /* Something related to rebuild. */
 121         uint64_t        rebuild_lba64;  /* Per-volume rebuild position. */
 122         uint32_t        magic_4;
 123         uint32_t        magic_5;
 124         uint32_t        filler3[325];
 125         uint32_t        checksum;
 126 } __packed;
 127
 128 struct g_raid_md_promise_perdisk {
 129         int              pd_updated;
 130         int              pd_subdisks;
 131         struct promise_raid_conf        *pd_meta[PROMISE_MAX_SUBDISKS];
 132 };
 133
 134 struct g_raid_md_promise_pervolume {
 135         struct promise_raid_conf        *pv_meta;
 136         uint64_t                         pv_id;
 137         uint16_t                         pv_generation;
 138         int                              pv_disks_present;
 139         int                              pv_started;
 140         struct callout                   pv_start_co;   /* STARTING state timer. */
 141 };
 142
 143 static g_raid_md_create_t g_raid_md_create_promise;
 144 static g_raid_md_taste_t g_raid_md_taste_promise;
 145 static g_raid_md_event_t g_raid_md_event_promise;
 146 static g_raid_md_volume_event_t g_raid_md_volume_event_promise;
 147 static g_raid_md_ctl_t g_raid_md_ctl_promise;
 148 static g_raid_md_write_t g_raid_md_write_promise;
 149 static g_raid_md_fail_disk_t g_raid_md_fail_disk_promise;
 150 static g_raid_md_free_disk_t g_raid_md_free_disk_promise;
 151 static g_raid_md_free_volume_t g_raid_md_free_volume_promise;
 152 static g_raid_md_free_t g_raid_md_free_promise;
 153
 154 static kobj_method_t g_raid_md_promise_methods[] = {
 155         KOBJMETHOD(g_raid_md_create,    g_raid_md_create_promise),
 156         KOBJMETHOD(g_raid_md_taste,     g_raid_md_taste_promise),
 157         KOBJMETHOD(g_raid_md_event,     g_raid_md_event_promise),
 158         KOBJMETHOD(g_raid_md_volume_event,      g_raid_md_volume_event_promise),
 159         KOBJMETHOD(g_raid_md_ctl,       g_raid_md_ctl_promise),
 160         KOBJMETHOD(g_raid_md_write,     g_raid_md_write_promise),
 161         KOBJMETHOD(g_raid_md_fail_disk, g_raid_md_fail_disk_promise),
 162         KOBJMETHOD(g_raid_md_free_disk, g_raid_md_free_disk_promise),
 163         KOBJMETHOD(g_raid_md_free_volume,       g_raid_md_free_volume_promise),
 164         KOBJMETHOD(g_raid_md_free,      g_raid_md_free_promise),
 165         { 0, 0 }
 166 };
 167
 168 static struct g_raid_md_class g_raid_md_promise_class = {
 169         "Promise",
 170         g_raid_md_promise_methods,
 171         sizeof(struct g_raid_md_object),
 172         .mdc_priority = 100
 173 };
 174
 175
 176 static void
 177 g_raid_md_promise_print(struct promise_raid_conf *meta)
 178 {
 179         int i;
 180
 181         if (g_raid_debug < 1)
 182                 return;
 183
 184         printf("********* ATA Promise Metadata *********\n");
 185         printf("promise_id          <%.24s>\n", meta->promise_id);
 186         printf("disk                %02x %02x %02x %02x %016jx\n",
 187             meta->disk.flags, meta->disk.number, meta->disk.channel,
 188             meta->disk.device, meta->disk.id);
 189         printf("disk_offset         %u\n", meta->disk_offset);
 190         printf("disk_sectors        %u\n", meta->disk_sectors);
 191         printf("rebuild_lba         %u\n", meta->rebuild_lba);
 192         printf("generation          %u\n", meta->generation);
 193         printf("status              0x%02x\n", meta->status);
 194         printf("type                %u\n", meta->type);
 195         printf("total_disks         %u\n", meta->total_disks);
 196         printf("stripe_shift        %u\n", meta->stripe_shift);
 197         printf("array_width         %u\n", meta->array_width);
 198         printf("array_number        %u\n", meta->array_number);
 199         printf("total_sectors       %u\n", meta->total_sectors);
 200         printf("cylinders           %u\n", meta->cylinders);
 201         printf("heads               %u\n", meta->heads);
 202         printf("sectors             %u\n", meta->sectors);
 203         printf("volume_id           0x%016jx\n", meta->volume_id);
 204         printf("disks:\n");
 205         for (i = 0; i < PROMISE_MAX_DISKS; i++ ) {
 206                 printf("                    %02x %02x %02x %02x %016jx\n",
 207                     meta->disks[i].flags, meta->disks[i].number,
 208                     meta->disks[i].channel, meta->disks[i].device,
 209                     meta->disks[i].id);
 210         }
 211         printf("name                <%.32s>\n", meta->name);
 212         printf("magic_3             0x%08x\n", meta->magic_3);
 213         printf("rebuild_lba64       %ju\n", meta->rebuild_lba64);
 214         printf("magic_4             0x%08x\n", meta->magic_4);
 215         printf("magic_5             0x%08x\n", meta->magic_5);
 216         printf("=================================================\n");
 217 }
 218
 219 static struct promise_raid_conf *
 220 promise_meta_copy(struct promise_raid_conf *meta)
 221 {
 222         struct promise_raid_conf *nmeta;
 223
 224         nmeta = malloc(sizeof(*nmeta), M_MD_PROMISE, M_WAITOK);
 225         memcpy(nmeta, meta, sizeof(*nmeta));
 226         return (nmeta);
 227 }
 228
 229 static int
 230 promise_meta_find_disk(struct promise_raid_conf *meta, uint64_t id)
 231 {
 232         int pos;
 233
 234         for (pos = 0; pos < meta->total_disks; pos++) {
 235                 if (meta->disks[pos].id == id)
 236                         return (pos);
 237         }
 238         return (-1);
 239 }
 240
 241 static int
 242 promise_meta_unused_range(struct promise_raid_conf **metaarr, int nsd,
 243     uint32_t sectors, uint32_t *off, uint32_t *size)
 244 {
 245         uint32_t coff, csize;
 246         int i, j;
 247
 248         sectors -= 131072;
 249         *off = 0;
 250         *size = 0;
 251         coff = 0;
 252         csize = sectors;
 253         i = 0;
 254         while (1) {
 255                 for (j = 0; j < nsd; j++) {
 256                         if (metaarr[j]->disk_offset >= coff) {
 257                                 csize = MIN(csize,
 258                                     metaarr[j]->disk_offset - coff);
 259                         }
 260                 }
 261                 if (csize > *size) {
 262                         *off = coff;
 263                         *size = csize;
 264                 }
 265                 if (i >= nsd)
 266                         break;
 267                 coff = metaarr[i]->disk_offset + metaarr[i]->disk_sectors;
 268                 csize = sectors - coff;
 269                 i++;
 270         };
 271         return ((*size > 0) ? 1 : 0);
 272 }
 273
 274 static int
 275 promise_meta_translate_disk(struct g_raid_volume *vol, int md_disk_pos)
 276 {
 277         int disk_pos, width;
 278
 279         if (md_disk_pos >= 0 && vol->v_raid_level == G_RAID_VOLUME_RL_RAID1E) {
 280                 width = vol->v_disks_count / 2;
 281                 disk_pos = (md_disk_pos / width) +
 282                     (md_disk_pos % width) * width;
 283         } else
 284                 disk_pos = md_disk_pos;
 285         return (disk_pos);
 286 }
 287
 288 static void
 289 promise_meta_get_name(struct promise_raid_conf *meta, char *buf)
 290 {
 291         int i;
 292
 293         strncpy(buf, meta->name, 32);
 294         buf[32] = 0;
 295         for (i = 31; i >= 0; i--) {
 296                 if (buf[i] > 0x20)
 297                         break;
 298                 buf[i] = 0;
 299         }
 300 }
 301
 302 static void
 303 promise_meta_put_name(struct promise_raid_conf *meta, char *buf)
 304 {
 305
 306         memset(meta->name, 0x20, 32);
 307         memcpy(meta->name, buf, MIN(strlen(buf), 32));
 308 }
 309
 310 static int
 311 promise_meta_read(struct g_consumer *cp, struct promise_raid_conf **metaarr)
 312 {
 313         struct g_provider *pp;
 314         struct promise_raid_conf *meta;
 315         char *buf;
 316         int error, i, subdisks;
 317         uint32_t checksum, *ptr;
 318
 319         pp = cp->provider;
 320         subdisks = 0;
 321 next:
 322         /* Read metadata block. */
 323         buf = g_read_data(cp, pp->mediasize - pp->sectorsize *
 324             (63 - subdisks * PROMISE_META_OFFSET),
 325             pp->sectorsize * 4, &error);
 326         if (buf == NULL) {
 327                 G_RAID_DEBUG(1, "Cannot read metadata from %s (error=%d).",
 328                     pp->name, error);
 329                 return (subdisks);
 330         }
 331         meta = (struct promise_raid_conf *)buf;
 332
 333         /* Check if this is an Promise RAID struct */
 334         if (strncmp(meta->promise_id, PROMISE_MAGIC, strlen(PROMISE_MAGIC)) &&
 335             strncmp(meta->promise_id, FREEBSD_MAGIC, strlen(FREEBSD_MAGIC))) {
 336                 if (subdisks == 0)
 337                         G_RAID_DEBUG(1,
 338                             "Promise signature check failed on %s", pp->name);
 339                 g_free(buf);
 340                 return (subdisks);
 341         }
 342         meta = malloc(sizeof(*meta), M_MD_PROMISE, M_WAITOK);
 343         memcpy(meta, buf, MIN(sizeof(*meta), pp->sectorsize * 4));
 344         g_free(buf);
 345
 346         /* Check metadata checksum. */
 347         for (checksum = 0, ptr = (uint32_t *)meta, i = 0; i < 511; i++)
 348                 checksum += *ptr++;
 349         if (checksum != meta->checksum) {
 350                 G_RAID_DEBUG(1, "Promise checksum check failed on %s", pp->name);
 351                 free(meta, M_MD_PROMISE);
 352                 return (subdisks);
 353         }
 354
 355         if ((meta->integrity & PROMISE_I_VALID) == 0) {
 356                 G_RAID_DEBUG(1, "Promise metadata is invalid on %s", pp->name);
 357                 free(meta, M_MD_PROMISE);
 358                 return (subdisks);
 359         }
 360
 361         if (meta->total_disks > PROMISE_MAX_DISKS) {
 362                 G_RAID_DEBUG(1, "Wrong number of disks on %s (%d)",
 363                     pp->name, meta->total_disks);
 364                 free(meta, M_MD_PROMISE);
 365                 return (subdisks);
 366         }
 367
 368         /* Save this part and look for next. */
 369         *metaarr = meta;
 370         metaarr++;
 371         subdisks++;
 372         if (subdisks < PROMISE_MAX_SUBDISKS)
 373                 goto next;
 374
 375         return (subdisks);
 376 }
 377
 378 static int
 379 promise_meta_write(struct g_consumer *cp,
 380     struct promise_raid_conf **metaarr, int nsd)
 381 {
 382         struct g_provider *pp;
 383         struct promise_raid_conf *meta;
 384         char *buf;
 385         int error, i, subdisk, fake;
 386         uint32_t checksum, *ptr, off, size;
 387
 388         pp = cp->provider;
 389         subdisk = 0;
 390         fake = 0;
 391 next:
 392         buf = malloc(pp->sectorsize * 4, M_MD_PROMISE, M_WAITOK | M_ZERO);
 393         meta = NULL;
 394         if (subdisk < nsd) {
 395                 meta = metaarr[subdisk];
 396         } else if (!fake && promise_meta_unused_range(metaarr, nsd,
 397             cp->provider->mediasize / cp->provider->sectorsize,
 398             &off, &size)) {
 399                 /* Optionally add record for unused space. */
 400                 meta = (struct promise_raid_conf *)buf;
 401                 memcpy(&meta->promise_id[0], PROMISE_MAGIC, sizeof(PROMISE_MAGIC));
 402                 meta->dummy_0 = 0x00020000;
 403                 meta->integrity = PROMISE_I_VALID;
 404                 meta->disk.flags = PROMISE_F_ONLINE | PROMISE_F_VALID;
 405                 meta->disk.number = 0xff;
 406                 arc4rand(&meta->disk.id, sizeof(meta->disk.id), 0);
 407                 meta->disk_offset = off;
 408                 meta->disk_sectors = size;
 409                 meta->rebuild_lba = UINT32_MAX;
 410                 fake = 1;
 411         }
 412         if (meta != NULL) {
 413                 /* Recalculate checksum for case if metadata were changed. */
 414                 meta->checksum = 0;
 415                 for (checksum = 0, ptr = (uint32_t *)meta, i = 0; i < 511; i++)
 416                         checksum += *ptr++;
 417                 meta->checksum = checksum;
 418                 memcpy(buf, meta, MIN(pp->sectorsize * 4, sizeof(*meta)));
 419         }
 420         error = g_write_data(cp, pp->mediasize - pp->sectorsize *
 421             (63 - subdisk * PROMISE_META_OFFSET),
 422             buf, pp->sectorsize * 4);
 423         if (error != 0) {
 424                 G_RAID_DEBUG(1, "Cannot write metadata to %s (error=%d).",
 425                     pp->name, error);
 426         }
 427         free(buf, M_MD_PROMISE);
 428
 429         subdisk++;
 430         if (subdisk < PROMISE_MAX_SUBDISKS)
 431                 goto next;
 432
 433         return (error);
 434 }
 435
 436 static int
 437 promise_meta_erase(struct g_consumer *cp)
 438 {
 439         struct g_provider *pp;
 440         char *buf;
 441         int error, subdisk;
 442
 443         pp = cp->provider;
 444         buf = malloc(4 * pp->sectorsize, M_MD_PROMISE, M_WAITOK | M_ZERO);
 445         for (subdisk = 0; subdisk < PROMISE_MAX_SUBDISKS; subdisk++) {
 446                 error = g_write_data(cp, pp->mediasize - pp->sectorsize *
 447                     (63 - subdisk * PROMISE_META_OFFSET),
 448                     buf, 4 * pp->sectorsize);
 449                 if (error != 0) {
 450                         G_RAID_DEBUG(1, "Cannot erase metadata on %s (error=%d).",
 451                             pp->name, error);
 452                 }
 453         }
 454         free(buf, M_MD_PROMISE);
 455         return (error);
 456 }
 457
 458 static int
 459 promise_meta_write_spare(struct g_consumer *cp)
 460 {
 461         struct promise_raid_conf *meta;
 462         int error;
 463
 464         meta = malloc(sizeof(*meta), M_MD_PROMISE, M_WAITOK | M_ZERO);
 465         memcpy(&meta->promise_id[0], PROMISE_MAGIC, sizeof(PROMISE_MAGIC));
 466         meta->dummy_0 = 0x00020000;
 467         meta->integrity = PROMISE_I_VALID;
 468         meta->disk.flags = PROMISE_F_SPARE | PROMISE_F_ONLINE | PROMISE_F_VALID;
 469         meta->disk.number = 0xff;
 470         arc4rand(&meta->disk.id, sizeof(meta->disk.id), 0);
 471         meta->disk_sectors = cp->provider->mediasize / cp->provider->sectorsize;
 472         meta->disk_sectors -= 131072;
 473         meta->rebuild_lba = UINT32_MAX;
 474         error = promise_meta_write(cp, &meta, 1);
 475         free(meta, M_MD_PROMISE);
 476         return (error);
 477 }
 478
 479 static struct g_raid_volume *
 480 g_raid_md_promise_get_volume(struct g_raid_softc *sc, uint64_t id)
 481 {
 482         struct g_raid_volume    *vol;
 483         struct g_raid_md_promise_pervolume *pv;
 484
 485         TAILQ_FOREACH(vol, &sc->sc_volumes, v_next) {
 486                 pv = vol->v_md_data;
 487                 if (pv->pv_id == id)
 488                         break;
 489         }
 490         return (vol);
 491 }
 492
 493 static int
 494 g_raid_md_promise_purge_volumes(struct g_raid_softc *sc)
 495 {
 496         struct g_raid_volume    *vol, *tvol;
 497         struct g_raid_md_promise_pervolume *pv;
 498         int i, res;
 499
 500         res = 0;
 501         TAILQ_FOREACH_SAFE(vol, &sc->sc_volumes, v_next, tvol) {
 502                 pv = vol->v_md_data;
 503                 if (!pv->pv_started || vol->v_stopping)
 504                         continue;
 505                 for (i = 0; i < vol->v_disks_count; i++) {
 506                         if (vol->v_subdisks[i].sd_state != G_RAID_SUBDISK_S_NONE)
 507                                 break;
 508                 }
 509                 if (i >= vol->v_disks_count) {
 510                         g_raid_destroy_volume(vol);
 511                         res = 1;
 512                 }
 513         }
 514         return (res);
 515 }
 516
 517 static int
 518 g_raid_md_promise_purge_disks(struct g_raid_softc *sc)
 519 {
 520         struct g_raid_disk      *disk, *tdisk;
 521         struct g_raid_volume    *vol;
 522         struct g_raid_md_promise_perdisk *pd;
 523         int i, j, res;
 524
 525         res = 0;
 526         TAILQ_FOREACH_SAFE(disk, &sc->sc_disks, d_next, tdisk) {
 527                 if (disk->d_state == G_RAID_DISK_S_SPARE)
 528                         continue;
 529                 pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
 530
 531                 /* Scan for deleted volumes. */
 532                 for (i = 0; i < pd->pd_subdisks; ) {
 533                         vol = g_raid_md_promise_get_volume(sc,
 534                             pd->pd_meta[i]->volume_id);
 535                         if (vol != NULL && !vol->v_stopping) {
 536                                 i++;
 537                                 continue;
 538                         }
 539                         free(pd->pd_meta[i], M_MD_PROMISE);
 540                         for (j = i; j < pd->pd_subdisks - 1; j++)
 541                                 pd->pd_meta[j] = pd->pd_meta[j + 1];
 542                         pd->pd_meta[PROMISE_MAX_SUBDISKS - 1] = NULL;
 543                         pd->pd_subdisks--;
 544                         pd->pd_updated = 1;
 545                 }
 546
 547                 /* If there is no metadata left - erase and delete disk. */
 548                 if (pd->pd_subdisks == 0) {
 549                         promise_meta_erase(disk->d_consumer);
 550                         g_raid_destroy_disk(disk);
 551                         res = 1;
 552                 }
 553         }
 554         return (res);
 555 }
 556
 557 static int
 558 g_raid_md_promise_supported(int level, int qual, int disks, int force)
 559 {
 560
 561         if (disks > PROMISE_MAX_DISKS)
 562                 return (0);
 563         switch (level) {
 564         case G_RAID_VOLUME_RL_RAID0:
 565                 if (disks < 1)
 566                         return (0);
 567                 if (!force && disks < 2)
 568                         return (0);
 569                 break;
 570         case G_RAID_VOLUME_RL_RAID1:
 571                 if (disks < 1)
 572                         return (0);
 573                 if (!force && (disks != 2))
 574                         return (0);
 575                 break;
 576         case G_RAID_VOLUME_RL_RAID1E:
 577                 if (disks < 2)
 578                         return (0);
 579                 if (disks % 2 != 0)
 580                         return (0);
 581                 if (!force && (disks != 4))
 582                         return (0);
 583                 break;
 584         case G_RAID_VOLUME_RL_SINGLE:
 585                 if (disks != 1)
 586                         return (0);
 587                 break;
 588         case G_RAID_VOLUME_RL_CONCAT:
 589                 if (disks < 2)
 590                         return (0);
 591                 break;
 592         case G_RAID_VOLUME_RL_RAID5:
 593                 if (disks < 3)
 594                         return (0);
 595                 break;
 596         default:
 597                 return (0);
 598         }
 599         if (qual != G_RAID_VOLUME_RLQ_NONE)
 600                 return (0);
 601         return (1);
 602 }
 603
 604 static int
 605 g_raid_md_promise_start_disk(struct g_raid_disk *disk, int sdn,
 606     struct g_raid_volume *vol)
 607 {
 608         struct g_raid_softc *sc;
 609         struct g_raid_subdisk *sd;
 610         struct g_raid_md_promise_perdisk *pd;
 611         struct g_raid_md_promise_pervolume *pv;
 612         struct promise_raid_conf *meta;
 613         off_t size;
 614         int disk_pos, md_disk_pos, i, resurrection = 0;
 615         uint32_t eoff, esize;
 616
 617         sc = disk->d_softc;
 618         pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
 619
 620         pv = vol->v_md_data;
 621         meta = pv->pv_meta;
 622
 623         if (sdn >= 0) {
 624                 /* Find disk position in metadata by it's serial. */
 625                 md_disk_pos = promise_meta_find_disk(meta, pd->pd_meta[sdn]->disk.id);
 626                 /* For RAID0+1 we need to translate order. */
 627                 disk_pos = promise_meta_translate_disk(vol, md_disk_pos);
 628         } else {
 629                 md_disk_pos = -1;
 630                 disk_pos = -1;
 631         }
 632         if (disk_pos < 0) {
 633                 G_RAID_DEBUG1(1, sc, "Disk %s is not part of the volume %s",
 634                     g_raid_get_diskname(disk), vol->v_name);
 635                 /* Failed stale disk is useless for us. */
 636                 if (sdn >= 0 &&
 637                     pd->pd_meta[sdn]->disk.flags & PROMISE_F_DOWN) {
 638                         g_raid_change_disk_state(disk, G_RAID_DISK_S_STALE_FAILED);
 639                         return (0);
 640                 }
 641                 /* If we were given specific metadata subdisk - erase it. */
 642                 if (sdn >= 0) {
 643                         free(pd->pd_meta[sdn], M_MD_PROMISE);
 644                         for (i = sdn; i < pd->pd_subdisks - 1; i++)
 645                                 pd->pd_meta[i] = pd->pd_meta[i + 1];
 646                         pd->pd_meta[PROMISE_MAX_SUBDISKS - 1] = NULL;
 647                         pd->pd_subdisks--;
 648                 }
 649                 /* If we are in the start process, that's all for now. */
 650                 if (!pv->pv_started)
 651                         goto nofit;
 652                 /*
 653                  * If we have already started - try to get use of the disk.
 654                  * Try to replace OFFLINE disks first, then FAILED.
 655                  */
 656                 promise_meta_unused_range(pd->pd_meta, pd->pd_subdisks,
 657                     disk->d_consumer->provider->mediasize /
 658                     disk->d_consumer->provider->sectorsize,
 659                     &eoff, &esize);
 660                 if (esize == 0) {
 661                         G_RAID_DEBUG1(1, sc, "No free space on disk %s",
 662                             g_raid_get_diskname(disk));
 663                         goto nofit;
 664                 }
 665                 size = INT64_MAX;
 666                 for (i = 0; i < vol->v_disks_count; i++) {
 667                         sd = &vol->v_subdisks[i];
 668                         if (sd->sd_state != G_RAID_SUBDISK_S_NONE)
 669                                 size = sd->sd_size;
 670                         if (sd->sd_state <= G_RAID_SUBDISK_S_FAILED &&
 671                             (disk_pos < 0 ||
 672                              vol->v_subdisks[i].sd_state < sd->sd_state))
 673                                 disk_pos = i;
 674                 }
 675                 if (disk_pos >= 0 &&
 676                     vol->v_raid_level != G_RAID_VOLUME_RL_CONCAT &&
 677                     (off_t)esize * 512 < size) {
 678                         G_RAID_DEBUG1(1, sc, "Disk %s free space "
 679                             "is too small (%ju < %ju)",
 680                             g_raid_get_diskname(disk),
 681                             (off_t)esize * 512, size);
 682                         disk_pos = -1;
 683                 }
 684                 if (disk_pos >= 0) {
 685                         if (vol->v_raid_level != G_RAID_VOLUME_RL_CONCAT)
 686                                 esize = size / 512;
 687                         /* For RAID0+1 we need to translate order. */
 688                         md_disk_pos = promise_meta_translate_disk(vol, disk_pos);
 689                 } else {
 690 nofit:
 691                         if (pd->pd_subdisks == 0) {
 692                                 g_raid_change_disk_state(disk,
 693                                     G_RAID_DISK_S_SPARE);
 694                         }
 695                         return (0);
 696                 }
 697                 G_RAID_DEBUG1(1, sc, "Disk %s takes pos %d in the volume %s",
 698                     g_raid_get_diskname(disk), disk_pos, vol->v_name);
 699                 resurrection = 1;
 700         }
 701
 702         sd = &vol->v_subdisks[disk_pos];
 703
 704         if (resurrection && sd->sd_disk != NULL) {
 705                 g_raid_change_disk_state(sd->sd_disk,
 706                     G_RAID_DISK_S_STALE_FAILED);
 707                 TAILQ_REMOVE(&sd->sd_disk->d_subdisks,
 708                     sd, sd_next);
 709         }
 710         vol->v_subdisks[disk_pos].sd_disk = disk;
 711         TAILQ_INSERT_TAIL(&disk->d_subdisks, sd, sd_next);
 712
 713         /* Welcome the new disk. */
 714         if (resurrection)
 715                 g_raid_change_disk_state(disk, G_RAID_DISK_S_ACTIVE);
 716         else if (meta->disks[md_disk_pos].flags & PROMISE_F_DOWN)
 717                 g_raid_change_disk_state(disk, G_RAID_DISK_S_FAILED);
 718         else
 719                 g_raid_change_disk_state(disk, G_RAID_DISK_S_ACTIVE);
 720
 721         if (resurrection) {
 722                 sd->sd_offset = (off_t)eoff * 512;
 723                 sd->sd_size = (off_t)esize * 512;
 724         } else {
 725                 sd->sd_offset = (off_t)pd->pd_meta[sdn]->disk_offset * 512;
 726                 sd->sd_size = (off_t)pd->pd_meta[sdn]->disk_sectors * 512;
 727         }
 728
 729         if (resurrection) {
 730                 /* Stale disk, almost same as new. */
 731                 g_raid_change_subdisk_state(sd,
 732                     G_RAID_SUBDISK_S_NEW);
 733         } else if (meta->disks[md_disk_pos].flags & PROMISE_F_DOWN) {
 734                 /* Failed disk. */
 735                 g_raid_change_subdisk_state(sd,
 736                     G_RAID_SUBDISK_S_FAILED);
 737         } else if (meta->disks[md_disk_pos].flags & PROMISE_F_REDIR) {
 738                 /* Rebuilding disk. */
 739                 g_raid_change_subdisk_state(sd,
 740                     G_RAID_SUBDISK_S_REBUILD);
 741                 if (pd->pd_meta[sdn]->generation != meta->generation)
 742                         sd->sd_rebuild_pos = 0;
 743                 else {
 744                         sd->sd_rebuild_pos =
 745                             (off_t)pd->pd_meta[sdn]->rebuild_lba * 512;
 746                 }
 747         } else if (!(meta->disks[md_disk_pos].flags & PROMISE_F_ONLINE)) {
 748                 /* Rebuilding disk. */
 749                 g_raid_change_subdisk_state(sd,
 750                     G_RAID_SUBDISK_S_NEW);
 751         } else if (pd->pd_meta[sdn]->generation != meta->generation ||
 752             (meta->status & PROMISE_S_MARKED)) {
 753                 /* Stale disk or dirty volume (unclean shutdown). */
 754                 g_raid_change_subdisk_state(sd,
 755                     G_RAID_SUBDISK_S_STALE);
 756         } else {
 757                 /* Up to date disk. */
 758                 g_raid_change_subdisk_state(sd,
 759                     G_RAID_SUBDISK_S_ACTIVE);
 760         }
 761         g_raid_event_send(sd, G_RAID_SUBDISK_E_NEW,
 762             G_RAID_EVENT_SUBDISK);
 763
 764         return (resurrection);
 765 }
 766
 767 static void
 768 g_raid_md_promise_refill(struct g_raid_softc *sc)
 769 {
 770         struct g_raid_volume *vol;
 771         struct g_raid_subdisk *sd;
 772         struct g_raid_disk *disk;
 773         struct g_raid_md_object *md;
 774         struct g_raid_md_promise_perdisk *pd;
 775         struct g_raid_md_promise_pervolume *pv;
 776         int update, updated, i, bad;
 777
 778         md = sc->sc_md;
 779 restart:
 780         updated = 0;
 781         TAILQ_FOREACH(vol, &sc->sc_volumes, v_next) {
 782                 pv = vol->v_md_data;
 783                 if (!pv->pv_started || vol->v_stopping)
 784                         continue;
 785
 786                 /* Search for subdisk that needs replacement. */
 787                 bad = 0;
 788                 for (i = 0; i < vol->v_disks_count; i++) {
 789                         sd = &vol->v_subdisks[i];
 790                         if (sd->sd_state == G_RAID_SUBDISK_S_NONE ||
 791                             sd->sd_state == G_RAID_SUBDISK_S_FAILED)
 792                                 bad = 1;
 793                 }
 794                 if (!bad)
 795                         continue;
 796
 797                 G_RAID_DEBUG1(1, sc, "Volume %s is not complete, "
 798                     "trying to refill.", vol->v_name);
 799
 800                 TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
 801                         /* Skip failed. */
 802                         if (disk->d_state < G_RAID_DISK_S_SPARE)
 803                                 continue;
 804                         /* Skip already used by this volume. */
 805                         for (i = 0; i < vol->v_disks_count; i++) {
 806                                 sd = &vol->v_subdisks[i];
 807                                 if (sd->sd_disk == disk)
 808                                         break;
 809                         }
 810                         if (i < vol->v_disks_count)
 811                                 continue;
 812
 813                         /* Try to use disk if it has empty extents. */
 814                         pd = disk->d_md_data;
 815                         if (pd->pd_subdisks < PROMISE_MAX_SUBDISKS) {
 816                                 update =
 817                                     g_raid_md_promise_start_disk(disk, -1, vol);
 818                         } else
 819                                 update = 0;
 820                         if (update) {
 821                                 g_raid_md_write_promise(md, vol, NULL, disk);
 822                                 break;
 823                         }
 824                         updated += update;
 825                 }
 826         }
 827         if (updated)
 828                 goto restart;
 829 }
 830
 831 static void
 832 g_raid_md_promise_start(struct g_raid_volume *vol)
 833 {
 834         struct g_raid_softc *sc;
 835         struct g_raid_subdisk *sd;
 836         struct g_raid_disk *disk;
 837         struct g_raid_md_object *md;
 838         struct g_raid_md_promise_perdisk *pd;
 839         struct g_raid_md_promise_pervolume *pv;
 840         struct promise_raid_conf *meta;
 841         int i;
 842
 843         sc = vol->v_softc;
 844         md = sc->sc_md;
 845         pv = vol->v_md_data;
 846         meta = pv->pv_meta;
 847
 848         if (meta->type == PROMISE_T_RAID0)
 849                 vol->v_raid_level = G_RAID_VOLUME_RL_RAID0;
 850         else if (meta->type == PROMISE_T_RAID1) {
 851                 if (meta->array_width == 1)
 852                         vol->v_raid_level = G_RAID_VOLUME_RL_RAID1;
 853                 else
 854                         vol->v_raid_level = G_RAID_VOLUME_RL_RAID1E;
 855         } else if (meta->type == PROMISE_T_RAID3)
 856                 vol->v_raid_level = G_RAID_VOLUME_RL_RAID3;
 857         else if (meta->type == PROMISE_T_RAID5)
 858                 vol->v_raid_level = G_RAID_VOLUME_RL_RAID5;
 859         else if (meta->type == PROMISE_T_SPAN)
 860                 vol->v_raid_level = G_RAID_VOLUME_RL_CONCAT;
 861         else if (meta->type == PROMISE_T_JBOD)
 862                 vol->v_raid_level = G_RAID_VOLUME_RL_SINGLE;
 863         else
 864                 vol->v_raid_level = G_RAID_VOLUME_RL_UNKNOWN;
 865         vol->v_raid_level_qualifier = G_RAID_VOLUME_RLQ_NONE;
 866         vol->v_strip_size = 512 << meta->stripe_shift; //ZZZ
 867         vol->v_disks_count = meta->total_disks;
 868         vol->v_mediasize = (off_t)meta->total_sectors * 512; //ZZZ
 869         vol->v_sectorsize = 512; //ZZZ
 870         for (i = 0; i < vol->v_disks_count; i++) {
 871                 sd = &vol->v_subdisks[i];
 872                 sd->sd_offset = (off_t)meta->disk_offset * 512; //ZZZ
 873                 sd->sd_size = (off_t)meta->disk_sectors * 512; //ZZZ
 874         }
 875         g_raid_start_volume(vol);
 876
 877         /* Make all disks found till the moment take their places. */
 878         TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
 879                 pd = disk->d_md_data;
 880                 for (i = 0; i < pd->pd_subdisks; i++) {
 881                         if (pd->pd_meta[i]->volume_id == meta->volume_id)
 882                                 g_raid_md_promise_start_disk(disk, i, vol);
 883                 }
 884         }
 885
 886         pv->pv_started = 1;
 887         callout_stop(&pv->pv_start_co);
 888         G_RAID_DEBUG1(0, sc, "Volume started.");
 889         g_raid_md_write_promise(md, vol, NULL, NULL);
 890
 891         /* Pickup any STALE/SPARE disks to refill array if needed. */
 892         g_raid_md_promise_refill(sc);
 893
 894         g_raid_event_send(vol, G_RAID_VOLUME_E_START, G_RAID_EVENT_VOLUME);
 895 }
 896
 897 static void
 898 g_raid_promise_go(void *arg)
 899 {
 900         struct g_raid_volume *vol;
 901         struct g_raid_softc *sc;
 902         struct g_raid_md_promise_pervolume *pv;
 903
 904         vol = arg;
 905         pv = vol->v_md_data;
 906         sc = vol->v_softc;
 907         if (!pv->pv_started) {
 908                 G_RAID_DEBUG1(0, sc, "Force volume start due to timeout.");
 909                 g_raid_event_send(vol, G_RAID_VOLUME_E_STARTMD,
 910                     G_RAID_EVENT_VOLUME);
 911         }
 912 }
 913
 914 static void
 915 g_raid_md_promise_new_disk(struct g_raid_disk *disk)
 916 {
 917         struct g_raid_softc *sc;
 918         struct g_raid_md_object *md;
 919         struct promise_raid_conf *pdmeta;
 920         struct g_raid_md_promise_perdisk *pd;
 921         struct g_raid_md_promise_pervolume *pv;
 922         struct g_raid_volume *vol;
 923         int i;
 924         char buf[33];
 925
 926         sc = disk->d_softc;
 927         md = sc->sc_md;
 928         pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
 929
 930         if (pd->pd_subdisks == 0) {
 931                 g_raid_change_disk_state(disk, G_RAID_DISK_S_SPARE);
 932                 g_raid_md_promise_refill(sc);
 933                 return;
 934         }
 935
 936         for (i = 0; i < pd->pd_subdisks; i++) {
 937                 pdmeta = pd->pd_meta[i];
 938
 939                 /* Look for volume with matching ID. */
 940                 vol = g_raid_md_promise_get_volume(sc, pdmeta->volume_id);
 941                 if (vol == NULL) {
 942                         promise_meta_get_name(pdmeta, buf);
 943                         vol = g_raid_create_volume(sc, buf, pdmeta->array_number);
 944                         pv = malloc(sizeof(*pv), M_MD_PROMISE, M_WAITOK | M_ZERO);
 945                         pv->pv_id = pdmeta->volume_id;
 946                         vol->v_md_data = pv;
 947                         callout_init(&pv->pv_start_co, 1);
 948                         callout_reset(&pv->pv_start_co,
 949                             g_raid_start_timeout * hz,
 950                             g_raid_promise_go, vol);
 951                 } else
 952                         pv = vol->v_md_data;
 953
 954                 /* If we haven't started yet - check metadata freshness. */
 955                 if (pv->pv_meta == NULL || !pv->pv_started) {
 956                         if (pv->pv_meta == NULL ||
 957                             ((int16_t)(pdmeta->generation - pv->pv_generation)) > 0) {
 958                                 G_RAID_DEBUG1(1, sc, "Newer disk");
 959                                 if (pv->pv_meta != NULL)
 960                                         free(pv->pv_meta, M_MD_PROMISE);
 961                                 pv->pv_meta = promise_meta_copy(pdmeta);
 962                                 pv->pv_generation = pv->pv_meta->generation;
 963                                 pv->pv_disks_present = 1;
 964                         } else if (pdmeta->generation == pv->pv_generation) {
 965                                 pv->pv_disks_present++;
 966                                 G_RAID_DEBUG1(1, sc, "Matching disk (%d of %d up)",
 967                                     pv->pv_disks_present,
 968                                     pv->pv_meta->total_disks);
 969                         } else {
 970                                 G_RAID_DEBUG1(1, sc, "Older disk");
 971                         }
 972                 }
 973         }
 974
 975         for (i = 0; i < pd->pd_subdisks; i++) {
 976                 pdmeta = pd->pd_meta[i];
 977
 978                 /* Look for volume with matching ID. */
 979                 vol = g_raid_md_promise_get_volume(sc, pdmeta->volume_id);
 980                 if (vol == NULL)
 981                         continue;
 982                 pv = vol->v_md_data;
 983
 984                 if (pv->pv_started) {
 985                         if (g_raid_md_promise_start_disk(disk, i, vol))
 986                                 g_raid_md_write_promise(md, vol, NULL, NULL);
 987                 } else {
 988                         /* If we collected all needed disks - start array. */
 989                         if (pv->pv_disks_present == pv->pv_meta->total_disks)
 990                                 g_raid_md_promise_start(vol);
 991                 }
 992         }
 993 }
 994
 995 static int
 996 g_raid_md_create_promise(struct g_raid_md_object *md, struct g_class *mp,
 997     struct g_geom **gp)
 998 {
 999         struct g_geom *geom;
1000         struct g_raid_softc *sc;
1001
1002         /* Search for existing node. */
1003         LIST_FOREACH(geom, &mp->geom, geom) {
1004                 sc = geom->softc;
1005                 if (sc == NULL)
1006                         continue;
1007                 if (sc->sc_stopping != 0)
1008                         continue;
1009                 if (sc->sc_md->mdo_class != md->mdo_class)
1010                         continue;
1011                 break;
1012         }
1013         if (geom != NULL) {
1014                 *gp = geom;
1015                 return (G_RAID_MD_TASTE_EXISTING);
1016         }
1017
1018         /* Create new one if not found. */
1019         sc = g_raid_create_node(mp, "Promise", md);
1020         if (sc == NULL)
1021                 return (G_RAID_MD_TASTE_FAIL);
1022         md->mdo_softc = sc;
1023         *gp = sc->sc_geom;
1024         return (G_RAID_MD_TASTE_NEW);
1025 }
1026
1027 static int
1028 g_raid_md_taste_promise(struct g_raid_md_object *md, struct g_class *mp,
1029                               struct g_consumer *cp, struct g_geom **gp)
1030 {
1031         struct g_consumer *rcp;
1032         struct g_provider *pp;
1033         struct g_raid_softc *sc;
1034         struct g_raid_disk *disk;
1035         struct promise_raid_conf *meta, *metaarr[4];
1036         struct g_raid_md_promise_perdisk *pd;
1037         struct g_geom *geom;
1038         int error, i, j, result, len, subdisks;
1039         char name[16];
1040         uint16_t vendor;
1041
1042         G_RAID_DEBUG(1, "Tasting Promise on %s", cp->provider->name);
1043         pp = cp->provider;
1044
1045         /* Read metadata from device. */
1046         meta = NULL;
1047         vendor = 0xffff;
1048         if (g_access(cp, 1, 0, 0) != 0)
1049                 return (G_RAID_MD_TASTE_FAIL);
1050         g_topology_unlock();
1051         len = 2;
1052         if (pp->geom->rank == 1)
1053                 g_io_getattr("GEOM::hba_vendor", cp, &len, &vendor);
1054         subdisks = promise_meta_read(cp, metaarr);
1055         g_topology_lock();
1056         g_access(cp, -1, 0, 0);
1057         if (subdisks == 0) {
1058                 if (g_raid_aggressive_spare) {
1059                         if (vendor == 0x105a || vendor == 0x1002) {
1060                                 G_RAID_DEBUG(1,
1061                                     "No Promise metadata, forcing spare.");
1062                                 goto search;
1063                         } else {
1064                                 G_RAID_DEBUG(1,
1065                                     "Promise/ATI vendor mismatch "
1066                                     "0x%04x != 0x105a/0x1002",
1067                                     vendor);
1068                         }
1069                 }
1070                 return (G_RAID_MD_TASTE_FAIL);
1071         }
1072
1073         /* Metadata valid. Print it. */
1074         for (i = 0; i < subdisks; i++)
1075                 g_raid_md_promise_print(metaarr[i]);
1076
1077         /* Purge meaningless (empty/spare) records. */
1078         for (i = 0; i < subdisks; ) {
1079                 if (metaarr[i]->disk.flags & PROMISE_F_ASSIGNED) {
1080                         i++;
1081                         continue;
1082                 }
1083                 free(metaarr[i], M_MD_PROMISE);
1084                 for (j = i; j < subdisks - 1; j++)
1085                         metaarr[i] = metaarr[j + 1];
1086                 metaarr[PROMISE_MAX_SUBDISKS - 1] = NULL;
1087                 subdisks--;
1088         }
1089
1090 search:
1091         /* Search for matching node. */
1092         sc = NULL;
1093         LIST_FOREACH(geom, &mp->geom, geom) {
1094                 sc = geom->softc;
1095                 if (sc == NULL)
1096                         continue;
1097                 if (sc->sc_stopping != 0)
1098                         continue;
1099                 if (sc->sc_md->mdo_class != md->mdo_class)
1100                         continue;
1101                 break;
1102         }
1103
1104         /* Found matching node. */
1105         if (geom != NULL) {
1106                 G_RAID_DEBUG(1, "Found matching array %s", sc->sc_name);
1107                 result = G_RAID_MD_TASTE_EXISTING;
1108
1109         } else { /* Not found matching node -- create one. */
1110                 result = G_RAID_MD_TASTE_NEW;
1111                 snprintf(name, sizeof(name), "Promise");
1112                 sc = g_raid_create_node(mp, name, md);
1113                 md->mdo_softc = sc;
1114                 geom = sc->sc_geom;
1115         }
1116
1117         rcp = g_new_consumer(geom);
1118         g_attach(rcp, pp);
1119         if (g_access(rcp, 1, 1, 1) != 0)
1120                 ; //goto fail1;
1121
1122         g_topology_unlock();
1123         sx_xlock(&sc->sc_lock);
1124
1125         pd = malloc(sizeof(*pd), M_MD_PROMISE, M_WAITOK | M_ZERO);
1126         pd->pd_subdisks = subdisks;
1127         for (i = 0; i < subdisks; i++)
1128                 pd->pd_meta[i] = metaarr[i];
1129         disk = g_raid_create_disk(sc);
1130         disk->d_md_data = (void *)pd;
1131         disk->d_consumer = rcp;
1132         rcp->private = disk;
1133
1134         /* Read kernel dumping information. */
1135         disk->d_kd.offset = 0;
1136         disk->d_kd.length = OFF_MAX;
1137         len = sizeof(disk->d_kd);
1138         error = g_io_getattr("GEOM::kerneldump", rcp, &len, &disk->d_kd);
1139         if (disk->d_kd.di.dumper == NULL)
1140                 G_RAID_DEBUG1(2, sc, "Dumping not supported by %s: %d.",
1141                     rcp->provider->name, error);
1142
1143         g_raid_md_promise_new_disk(disk);
1144
1145         sx_xunlock(&sc->sc_lock);
1146         g_topology_lock();
1147         *gp = geom;
1148         return (result);
1149 }
1150
1151 static int
1152 g_raid_md_event_promise(struct g_raid_md_object *md,
1153     struct g_raid_disk *disk, u_int event)
1154 {
1155         struct g_raid_softc *sc;
1156         struct g_raid_md_promise_perdisk *pd;
1157
1158         sc = md->mdo_softc;
1159         if (disk == NULL)
1160                 return (-1);
1161         pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
1162         switch (event) {
1163         case G_RAID_DISK_E_DISCONNECTED:
1164                 /* Delete disk. */
1165                 g_raid_change_disk_state(disk, G_RAID_DISK_S_NONE);
1166                 g_raid_destroy_disk(disk);
1167                 g_raid_md_promise_purge_volumes(sc);
1168
1169                 /* Write updated metadata to all disks. */
1170                 g_raid_md_write_promise(md, NULL, NULL, NULL);
1171
1172                 /* Check if anything left. */
1173                 if (g_raid_ndisks(sc, -1) == 0)
1174                         g_raid_destroy_node(sc, 0);
1175                 else
1176                         g_raid_md_promise_refill(sc);
1177                 return (0);
1178         }
1179         return (-2);
1180 }
1181
1182 static int
1183 g_raid_md_volume_event_promise(struct g_raid_md_object *md,
1184     struct g_raid_volume *vol, u_int event)
1185 {
1186         struct g_raid_softc *sc;
1187         struct g_raid_md_promise_pervolume *pv;
1188
1189         sc = md->mdo_softc;
1190         pv = (struct g_raid_md_promise_pervolume *)vol->v_md_data;
1191         switch (event) {
1192         case G_RAID_VOLUME_E_STARTMD:
1193                 if (!pv->pv_started)
1194                         g_raid_md_promise_start(vol);
1195                 return (0);
1196         }
1197         return (-2);
1198 }
1199
1200 static int
1201 g_raid_md_ctl_promise(struct g_raid_md_object *md,
1202     struct gctl_req *req)
1203 {
1204         struct g_raid_softc *sc;
1205         struct g_raid_volume *vol, *vol1;
1206         struct g_raid_subdisk *sd;
1207         struct g_raid_disk *disk, *disks[PROMISE_MAX_DISKS];
1208         struct g_raid_md_promise_perdisk *pd;
1209         struct g_raid_md_promise_pervolume *pv;
1210         struct g_consumer *cp;
1211         struct g_provider *pp;
1212         char arg[16];
1213         const char *verb, *volname, *levelname, *diskname;
1214         char *tmp;
1215         int *nargs, *force;
1216         off_t size, sectorsize, strip;
1217         intmax_t *sizearg, *striparg;
1218         uint32_t offs[PROMISE_MAX_DISKS], esize;
1219         int numdisks, i, len, level, qual;
1220         int error;
1221
1222         sc = md->mdo_softc;
1223         verb = gctl_get_param(req, "verb", NULL);
1224         nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
1225         error = 0;
1226         if (strcmp(verb, "label") == 0) {
1227
1228                 if (*nargs < 4) {
1229                         gctl_error(req, "Invalid number of arguments.");
1230                         return (-1);
1231                 }
1232                 volname = gctl_get_asciiparam(req, "arg1");
1233                 if (volname == NULL) {
1234                         gctl_error(req, "No volume name.");
1235                         return (-2);
1236                 }
1237                 levelname = gctl_get_asciiparam(req, "arg2");
1238                 if (levelname == NULL) {
1239                         gctl_error(req, "No RAID level.");
1240                         return (-3);
1241                 }
1242                 if (g_raid_volume_str2level(levelname, &level, &qual)) {
1243                         gctl_error(req, "Unknown RAID level '%s'.", levelname);
1244                         return (-4);
1245                 }
1246                 numdisks = *nargs - 3;
1247                 force = gctl_get_paraml(req, "force", sizeof(*force));
1248                 if (!g_raid_md_promise_supported(level, qual, numdisks,
1249                     force ? *force : 0)) {
1250                         gctl_error(req, "Unsupported RAID level "
1251                             "(0x%02x/0x%02x), or number of disks (%d).",
1252                             level, qual, numdisks);
1253                         return (-5);
1254                 }
1255
1256                 /* Search for disks, connect them and probe. */
1257                 size = INT64_MAX;
1258                 sectorsize = 0;
1259                 bzero(disks, sizeof(disks));
1260                 bzero(offs, sizeof(offs));
1261                 for (i = 0; i < numdisks; i++) {
1262                         snprintf(arg, sizeof(arg), "arg%d", i + 3);
1263                         diskname = gctl_get_asciiparam(req, arg);
1264                         if (diskname == NULL) {
1265                                 gctl_error(req, "No disk name (%s).", arg);
1266                                 error = -6;
1267                                 break;
1268                         }
1269                         if (strcmp(diskname, "NONE") == 0)
1270                                 continue;
1271
1272                         TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
1273                                 if (disk->d_consumer != NULL &&
1274                                     disk->d_consumer->provider != NULL &&
1275                                     strcmp(disk->d_consumer->provider->name,
1276                                      diskname) == 0)
1277                                         break;
1278                         }
1279                         if (disk != NULL) {
1280                                 if (disk->d_state != G_RAID_DISK_S_ACTIVE) {
1281                                         gctl_error(req, "Disk '%s' is in a "
1282                                             "wrong state (%s).", diskname,
1283                                             g_raid_disk_state2str(disk->d_state));
1284                                         error = -7;
1285                                         break;
1286                                 }
1287                                 pd = disk->d_md_data;
1288                                 if (pd->pd_subdisks >= PROMISE_MAX_SUBDISKS) {
1289                                         gctl_error(req, "Disk '%s' already "
1290                                             "used by %d volumes.",
1291                                             diskname, pd->pd_subdisks);
1292                                         error = -7;
1293                                         break;
1294                                 }
1295                                 pp = disk->d_consumer->provider;
1296                                 disks[i] = disk;
1297                                 promise_meta_unused_range(pd->pd_meta,
1298                                     pd->pd_subdisks,
1299                                     pp->mediasize / pp->sectorsize,
1300                                     &offs[i], &esize);
1301                                 size = MIN(size, (off_t)esize * pp->sectorsize);
1302                                 sectorsize = MAX(sectorsize, pp->sectorsize);
1303                                 continue;
1304                         }
1305
1306                         g_topology_lock();
1307                         cp = g_raid_open_consumer(sc, diskname);
1308                         if (cp == NULL) {
1309                                 gctl_error(req, "Can't open disk '%s'.",
1310                                     diskname);
1311                                 g_topology_unlock();
1312                                 error = -8;
1313                                 break;
1314                         }
1315                         pp = cp->provider;
1316                         pd = malloc(sizeof(*pd), M_MD_PROMISE, M_WAITOK | M_ZERO);
1317                         disk = g_raid_create_disk(sc);
1318                         disk->d_md_data = (void *)pd;
1319                         disk->d_consumer = cp;
1320                         disks[i] = disk;
1321                         cp->private = disk;
1322                         g_topology_unlock();
1323
1324                         /* Read kernel dumping information. */
1325                         disk->d_kd.offset = 0;
1326                         disk->d_kd.length = OFF_MAX;
1327                         len = sizeof(disk->d_kd);
1328                         g_io_getattr("GEOM::kerneldump", cp, &len, &disk->d_kd);
1329                         if (disk->d_kd.di.dumper == NULL)
1330                                 G_RAID_DEBUG1(2, sc,
1331                                     "Dumping not supported by %s.",
1332                                     cp->provider->name);
1333
1334                         /* Reserve some space for metadata. */
1335                         size = MIN(size, pp->mediasize - 131072llu * pp->sectorsize);
1336                         sectorsize = MAX(sectorsize, pp->sectorsize);
1337                 }
1338                 if (error != 0) {
1339                         for (i = 0; i < numdisks; i++) {
1340                                 if (disks[i] != NULL &&
1341                                     disks[i]->d_state == G_RAID_DISK_S_NONE)
1342                                         g_raid_destroy_disk(disks[i]);
1343                         }
1344                         return (error);
1345                 }
1346
1347                 /* Handle size argument. */
1348                 len = sizeof(*sizearg);
1349                 sizearg = gctl_get_param(req, "size", &len);
1350                 if (sizearg != NULL && len == sizeof(*sizearg) &&
1351                     *sizearg > 0) {
1352                         if (*sizearg > size) {
1353                                 gctl_error(req, "Size too big %lld > %lld.",
1354                                     (long long)*sizearg, (long long)size);
1355                                 return (-9);
1356                         }
1357                         size = *sizearg;
1358                 }
1359
1360                 /* Handle strip argument. */
1361                 strip = 131072;
1362                 len = sizeof(*striparg);
1363                 striparg = gctl_get_param(req, "strip", &len);
1364                 if (striparg != NULL && len == sizeof(*striparg) &&
1365                     *striparg > 0) {
1366                         if (*striparg < sectorsize) {
1367                                 gctl_error(req, "Strip size too small.");
1368                                 return (-10);
1369                         }
1370                         if (*striparg % sectorsize != 0) {
1371                                 gctl_error(req, "Incorrect strip size.");
1372                                 return (-11);
1373                         }
1374                         strip = *striparg;
1375                 }
1376
1377                 /* Round size down to strip or sector. */
1378                 if (level == G_RAID_VOLUME_RL_RAID1 ||
1379                     level == G_RAID_VOLUME_RL_SINGLE ||
1380                     level == G_RAID_VOLUME_RL_CONCAT)
1381                         size -= (size % sectorsize);
1382                 else if (level == G_RAID_VOLUME_RL_RAID1E &&
1383                     (numdisks & 1) != 0)
1384                         size -= (size % (2 * strip));
1385                 else
1386                         size -= (size % strip);
1387                 if (size <= 0) {
1388                         gctl_error(req, "Size too small.");
1389                         return (-13);
1390                 }
1391                 if (size > 0xffffffffllu * sectorsize) {
1392                         gctl_error(req, "Size too big.");
1393                         return (-14);
1394                 }
1395
1396                 /* We have all we need, create things: volume, ... */
1397                 pv = malloc(sizeof(*pv), M_MD_PROMISE, M_WAITOK | M_ZERO);
1398                 arc4rand(&pv->pv_id, sizeof(pv->pv_id), 0);
1399                 pv->pv_generation = 0;
1400                 pv->pv_started = 1;
1401                 vol = g_raid_create_volume(sc, volname, -1);
1402                 vol->v_md_data = pv;
1403                 vol->v_raid_level = level;
1404                 vol->v_raid_level_qualifier = G_RAID_VOLUME_RLQ_NONE;
1405                 vol->v_strip_size = strip;
1406                 vol->v_disks_count = numdisks;
1407                 if (level == G_RAID_VOLUME_RL_RAID0 ||
1408                     level == G_RAID_VOLUME_RL_CONCAT ||
1409                     level == G_RAID_VOLUME_RL_SINGLE)
1410                         vol->v_mediasize = size * numdisks;
1411                 else if (level == G_RAID_VOLUME_RL_RAID1)
1412                         vol->v_mediasize = size;
1413                 else if (level == G_RAID_VOLUME_RL_RAID3 ||
1414                     level == G_RAID_VOLUME_RL_RAID5)
1415                         vol->v_mediasize = size * (numdisks - 1);
1416                 else { /* RAID1E */
1417                         vol->v_mediasize = ((size * numdisks) / strip / 2) *
1418                             strip;
1419                 }
1420                 vol->v_sectorsize = sectorsize;
1421                 g_raid_start_volume(vol);
1422
1423                 /* , and subdisks. */
1424                 for (i = 0; i < numdisks; i++) {
1425                         disk = disks[i];
1426                         sd = &vol->v_subdisks[i];
1427                         sd->sd_disk = disk;
1428                         sd->sd_offset = (off_t)offs[i] * 512;
1429                         sd->sd_size = size;
1430                         if (disk == NULL)
1431                                 continue;
1432                         TAILQ_INSERT_TAIL(&disk->d_subdisks, sd, sd_next);
1433                         g_raid_change_disk_state(disk,
1434                             G_RAID_DISK_S_ACTIVE);
1435                         g_raid_change_subdisk_state(sd,
1436                             G_RAID_SUBDISK_S_ACTIVE);
1437                         g_raid_event_send(sd, G_RAID_SUBDISK_E_NEW,
1438                             G_RAID_EVENT_SUBDISK);
1439                 }
1440
1441                 /* Write metadata based on created entities. */
1442                 G_RAID_DEBUG1(0, sc, "Array started.");
1443                 g_raid_md_write_promise(md, vol, NULL, NULL);
1444
1445                 /* Pickup any STALE/SPARE disks to refill array if needed. */
1446                 g_raid_md_promise_refill(sc);
1447
1448                 g_raid_event_send(vol, G_RAID_VOLUME_E_START,
1449                     G_RAID_EVENT_VOLUME);
1450                 return (0);
1451         }
1452         if (strcmp(verb, "add") == 0) {
1453
1454                 gctl_error(req, "`add` command is not applicable, "
1455                     "use `label` instead.");
1456                 return (-99);
1457         }
1458         if (strcmp(verb, "delete") == 0) {
1459
1460                 /* Full node destruction. */
1461                 if (*nargs == 1) {
1462                         /* Check if some volume is still open. */
1463                         force = gctl_get_paraml(req, "force", sizeof(*force));
1464                         if (force != NULL && *force == 0 &&
1465                             g_raid_nopens(sc) != 0) {
1466                                 gctl_error(req, "Some volume is still open.");
1467                                 return (-4);
1468                         }
1469
1470                         TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
1471                                 if (disk->d_consumer)
1472                                         promise_meta_erase(disk->d_consumer);
1473                         }
1474                         g_raid_destroy_node(sc, 0);
1475                         return (0);
1476                 }
1477
1478                 /* Destroy specified volume. If it was last - all node. */
1479                 if (*nargs != 2) {
1480                         gctl_error(req, "Invalid number of arguments.");
1481                         return (-1);
1482                 }
1483                 volname = gctl_get_asciiparam(req, "arg1");
1484                 if (volname == NULL) {
1485                         gctl_error(req, "No volume name.");
1486                         return (-2);
1487                 }
1488
1489                 /* Search for volume. */
1490                 TAILQ_FOREACH(vol, &sc->sc_volumes, v_next) {
1491                         if (strcmp(vol->v_name, volname) == 0)
1492                                 break;
1493                 }
1494                 if (vol == NULL) {
1495                         i = strtol(volname, &tmp, 10);
1496                         if (verb != volname && tmp[0] == 0) {
1497                                 TAILQ_FOREACH(vol, &sc->sc_volumes, v_next) {
1498                                         if (vol->v_global_id == i)
1499                                                 break;
1500                                 }
1501                         }
1502                 }
1503                 if (vol == NULL) {
1504                         gctl_error(req, "Volume '%s' not found.", volname);
1505                         return (-3);
1506                 }
1507
1508                 /* Check if volume is still open. */
1509                 force = gctl_get_paraml(req, "force", sizeof(*force));
1510                 if (force != NULL && *force == 0 &&
1511                     vol->v_provider_open != 0) {
1512                         gctl_error(req, "Volume is still open.");
1513                         return (-4);
1514                 }
1515
1516                 /* Destroy volume and potentially node. */
1517                 i = 0;
1518                 TAILQ_FOREACH(vol1, &sc->sc_volumes, v_next)
1519                         i++;
1520                 if (i >= 2) {
1521                         g_raid_destroy_volume(vol);
1522                         g_raid_md_promise_purge_disks(sc);
1523                         g_raid_md_write_promise(md, NULL, NULL, NULL);
1524                 } else {
1525                         TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
1526                                 if (disk->d_consumer)
1527                                         promise_meta_erase(disk->d_consumer);
1528                         }
1529                         g_raid_destroy_node(sc, 0);
1530                 }
1531                 return (0);
1532         }
1533         if (strcmp(verb, "remove") == 0 ||
1534             strcmp(verb, "fail") == 0) {
1535                 if (*nargs < 2) {
1536                         gctl_error(req, "Invalid number of arguments.");
1537                         return (-1);
1538                 }
1539                 for (i = 1; i < *nargs; i++) {
1540                         snprintf(arg, sizeof(arg), "arg%d", i);
1541                         diskname = gctl_get_asciiparam(req, arg);
1542                         if (diskname == NULL) {
1543                                 gctl_error(req, "No disk name (%s).", arg);
1544                                 error = -2;
1545                                 break;
1546                         }
1547                         if (strncmp(diskname, "/dev/", 5) == 0)
1548                                 diskname += 5;
1549
1550                         TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
1551                                 if (disk->d_consumer != NULL &&
1552                                     disk->d_consumer->provider != NULL &&
1553                                     strcmp(disk->d_consumer->provider->name,
1554                                      diskname) == 0)
1555                                         break;
1556                         }
1557                         if (disk == NULL) {
1558                                 gctl_error(req, "Disk '%s' not found.",
1559                                     diskname);
1560                                 error = -3;
1561                                 break;
1562                         }
1563
1564                         if (strcmp(verb, "fail") == 0) {
1565                                 g_raid_md_fail_disk_promise(md, NULL, disk);
1566                                 continue;
1567                         }
1568
1569                         pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
1570
1571                         /* Erase metadata on deleting disk and destroy it. */
1572                         promise_meta_erase(disk->d_consumer);
1573                         g_raid_destroy_disk(disk);
1574                 }
1575                 g_raid_md_promise_purge_volumes(sc);
1576
1577                 /* Write updated metadata to remaining disks. */
1578                 g_raid_md_write_promise(md, NULL, NULL, NULL);
1579
1580                 /* Check if anything left. */
1581                 if (g_raid_ndisks(sc, -1) == 0)
1582                         g_raid_destroy_node(sc, 0);
1583                 else
1584                         g_raid_md_promise_refill(sc);
1585                 return (error);
1586         }
1587         if (strcmp(verb, "insert") == 0) {
1588                 if (*nargs < 2) {
1589                         gctl_error(req, "Invalid number of arguments.");
1590                         return (-1);
1591                 }
1592                 for (i = 1; i < *nargs; i++) {
1593                         /* Get disk name. */
1594                         snprintf(arg, sizeof(arg), "arg%d", i);
1595                         diskname = gctl_get_asciiparam(req, arg);
1596                         if (diskname == NULL) {
1597                                 gctl_error(req, "No disk name (%s).", arg);
1598                                 error = -3;
1599                                 break;
1600                         }
1601
1602                         /* Try to find provider with specified name. */
1603                         g_topology_lock();
1604                         cp = g_raid_open_consumer(sc, diskname);
1605                         if (cp == NULL) {
1606                                 gctl_error(req, "Can't open disk '%s'.",
1607                                     diskname);
1608                                 g_topology_unlock();
1609                                 error = -4;
1610                                 break;
1611                         }
1612                         pp = cp->provider;
1613                         g_topology_unlock();
1614
1615                         pd = malloc(sizeof(*pd), M_MD_PROMISE, M_WAITOK | M_ZERO);
1616
1617                         disk = g_raid_create_disk(sc);
1618                         disk->d_consumer = cp;
1619                         disk->d_consumer->private = disk;
1620                         disk->d_md_data = (void *)pd;
1621                         cp->private = disk;
1622
1623                         /* Read kernel dumping information. */
1624                         disk->d_kd.offset = 0;
1625                         disk->d_kd.length = OFF_MAX;
1626                         len = sizeof(disk->d_kd);
1627                         g_io_getattr("GEOM::kerneldump", cp, &len, &disk->d_kd);
1628                         if (disk->d_kd.di.dumper == NULL)
1629                                 G_RAID_DEBUG1(2, sc,
1630                                     "Dumping not supported by %s.",
1631                                     cp->provider->name);
1632
1633                         /* Welcome the "new" disk. */
1634                         g_raid_change_disk_state(disk, G_RAID_DISK_S_SPARE);
1635                         promise_meta_write_spare(cp);
1636                         g_raid_md_promise_refill(sc);
1637                 }
1638                 return (error);
1639         }
1640         return (-100);
1641 }
1642
1643 static int
1644 g_raid_md_write_promise(struct g_raid_md_object *md, struct g_raid_volume *tvol,
1645     struct g_raid_subdisk *tsd, struct g_raid_disk *tdisk)
1646 {
1647         struct g_raid_softc *sc;
1648         struct g_raid_volume *vol;
1649         struct g_raid_subdisk *sd;
1650         struct g_raid_disk *disk;
1651         struct g_raid_md_promise_perdisk *pd;
1652         struct g_raid_md_promise_pervolume *pv;
1653         struct promise_raid_conf *meta;
1654         off_t rebuild_lba64;
1655         int i, j, pos, rebuild;
1656
1657         sc = md->mdo_softc;
1658
1659         if (sc->sc_stopping == G_RAID_DESTROY_HARD)
1660                 return (0);
1661
1662         /* Generate new per-volume metadata for affected volumes. */
1663         TAILQ_FOREACH(vol, &sc->sc_volumes, v_next) {
1664                 if (vol->v_stopping)
1665                         continue;
1666
1667                 /* Skip volumes not related to specified targets. */
1668                 if (tvol != NULL && vol != tvol)
1669                         continue;
1670                 if (tsd != NULL && vol != tsd->sd_volume)
1671                         continue;
1672                 if (tdisk != NULL) {
1673                         for (i = 0; i < vol->v_disks_count; i++) {
1674                                 if (vol->v_subdisks[i].sd_disk == tdisk)
1675                                         break;
1676                         }
1677                         if (i >= vol->v_disks_count)
1678                                 continue;
1679                 }
1680
1681                 pv = (struct g_raid_md_promise_pervolume *)vol->v_md_data;
1682                 pv->pv_generation++;
1683
1684                 meta = malloc(sizeof(*meta), M_MD_PROMISE, M_WAITOK | M_ZERO);
1685                 if (pv->pv_meta != NULL)
1686                         memcpy(meta, pv->pv_meta, sizeof(*meta));
1687                 memcpy(meta->promise_id, PROMISE_MAGIC, sizeof(PROMISE_MAGIC));
1688                 meta->dummy_0 = 0x00020000;
1689                 meta->integrity = PROMISE_I_VALID;
1690
1691                 meta->generation = pv->pv_generation;
1692                 meta->status = PROMISE_S_VALID | PROMISE_S_ONLINE |
1693                     PROMISE_S_INITED | PROMISE_S_READY;
1694                 if (vol->v_state <= G_RAID_VOLUME_S_DEGRADED)
1695                         meta->status |= PROMISE_S_DEGRADED;
1696                 if (vol->v_dirty)
1697                         meta->status |= PROMISE_S_MARKED; /* XXX: INVENTED! */
1698                 if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID0 ||
1699                     vol->v_raid_level == G_RAID_VOLUME_RL_SINGLE)
1700                         meta->type = PROMISE_T_RAID0;
1701                 else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID1 ||
1702                     vol->v_raid_level == G_RAID_VOLUME_RL_RAID1E)
1703                         meta->type = PROMISE_T_RAID1;
1704                 else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID3)
1705                         meta->type = PROMISE_T_RAID3;
1706                 else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID5)
1707                         meta->type = PROMISE_T_RAID5;
1708                 else if (vol->v_raid_level == G_RAID_VOLUME_RL_CONCAT)
1709                         meta->type = PROMISE_T_SPAN;
1710                 else
1711                         meta->type = PROMISE_T_JBOD;
1712                 meta->total_disks = vol->v_disks_count;
1713                 meta->stripe_shift = ffs(vol->v_strip_size / 1024);
1714                 meta->array_width = vol->v_disks_count;
1715                 if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID1 ||
1716                     vol->v_raid_level == G_RAID_VOLUME_RL_RAID1E)
1717                         meta->array_width /= 2;
1718                 meta->array_number = vol->v_global_id;
1719                 meta->total_sectors = vol->v_mediasize / vol->v_sectorsize;
1720                 meta->cylinders = meta->total_sectors / (255 * 63) - 1;
1721                 meta->heads = 254;
1722                 meta->sectors = 63;
1723                 meta->volume_id = pv->pv_id;
1724                 rebuild_lba64 = UINT64_MAX;
1725                 rebuild = 0;
1726                 for (i = 0; i < vol->v_disks_count; i++) {
1727                         sd = &vol->v_subdisks[i];
1728                         /* For RAID0+1 we need to translate order. */
1729                         pos = promise_meta_translate_disk(vol, i);
1730                         meta->disks[pos].flags = PROMISE_F_VALID |
1731                             PROMISE_F_ASSIGNED;
1732                         if (sd->sd_state == G_RAID_SUBDISK_S_NONE) {
1733                                 meta->disks[pos].flags |= 0;
1734                         } else if (sd->sd_state == G_RAID_SUBDISK_S_FAILED) {
1735                                 meta->disks[pos].flags |=
1736                                     PROMISE_F_DOWN | PROMISE_F_REDIR;
1737                         } else if (sd->sd_state <= G_RAID_SUBDISK_S_REBUILD) {
1738                                 meta->disks[pos].flags |=
1739                                     PROMISE_F_ONLINE | PROMISE_F_REDIR;
1740                                 if (sd->sd_state == G_RAID_SUBDISK_S_REBUILD) {
1741                                         rebuild_lba64 = MIN(rebuild_lba64,
1742                                             sd->sd_rebuild_pos / 512);
1743                                 } else
1744                                         rebuild_lba64 = 0;
1745                                 rebuild = 1;
1746                         } else {
1747                                 meta->disks[pos].flags |= PROMISE_F_ONLINE;
1748                                 if (sd->sd_state < G_RAID_SUBDISK_S_ACTIVE) {
1749                                         meta->status |= PROMISE_S_MARKED;
1750                                         if (sd->sd_state == G_RAID_SUBDISK_S_RESYNC) {
1751                                                 rebuild_lba64 = MIN(rebuild_lba64,
1752                                                     sd->sd_rebuild_pos / 512);
1753                                         } else
1754                                                 rebuild_lba64 = 0;
1755                                 }
1756                         }
1757                         if (pv->pv_meta != NULL) {
1758                                 meta->disks[pos].id = pv->pv_meta->disks[pos].id;
1759                         } else {
1760                                 meta->disks[pos].number = i * 2;
1761                                 arc4rand(&meta->disks[pos].id,
1762                                     sizeof(meta->disks[pos].id), 0);
1763                         }
1764                 }
1765                 promise_meta_put_name(meta, vol->v_name);
1766
1767                 /* Try to mimic AMD BIOS rebuild/resync behavior. */
1768                 if (rebuild_lba64 != UINT64_MAX) {
1769                         if (rebuild)
1770                                 meta->magic_3 = 0x03040010UL; /* Rebuild? */
1771                         else
1772                                 meta->magic_3 = 0x03040008UL; /* Resync? */
1773                         /* Translate from per-disk to per-volume LBA. */
1774                         if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID1 ||
1775                             vol->v_raid_level == G_RAID_VOLUME_RL_RAID1E) {
1776                                 rebuild_lba64 *= meta->array_width;
1777                         } else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID3 ||
1778                             vol->v_raid_level == G_RAID_VOLUME_RL_RAID5) {
1779                                 rebuild_lba64 *= meta->array_width - 1;
1780                         } else
1781                                 rebuild_lba64 = 0;
1782                 } else
1783                         meta->magic_3 = 0x03000000UL;
1784                 meta->rebuild_lba64 = rebuild_lba64;
1785                 meta->magic_4 = 0x04010101UL;
1786
1787                 /* Replace per-volume metadata with new. */
1788                 if (pv->pv_meta != NULL)
1789                         free(pv->pv_meta, M_MD_PROMISE);
1790                 pv->pv_meta = meta;
1791
1792                 /* Copy new metadata to the disks, adding or replacing old. */
1793                 for (i = 0; i < vol->v_disks_count; i++) {
1794                         sd = &vol->v_subdisks[i];
1795                         disk = sd->sd_disk;
1796                         if (disk == NULL)
1797                                 continue;
1798                         /* For RAID0+1 we need to translate order. */
1799                         pos = promise_meta_translate_disk(vol, i);
1800                         pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
1801                         for (j = 0; j < pd->pd_subdisks; j++) {
1802                                 if (pd->pd_meta[j]->volume_id == meta->volume_id)
1803                                         break;
1804                         }
1805                         if (j == pd->pd_subdisks)
1806                                 pd->pd_subdisks++;
1807                         if (pd->pd_meta[j] != NULL)
1808                                 free(pd->pd_meta[j], M_MD_PROMISE);
1809                         pd->pd_meta[j] = promise_meta_copy(meta);
1810                         pd->pd_meta[j]->disk = meta->disks[pos];
1811                         pd->pd_meta[j]->disk.number = pos;
1812                         pd->pd_meta[j]->disk_offset = sd->sd_offset / 512;
1813                         pd->pd_meta[j]->disk_sectors = sd->sd_size / 512;
1814                         if (sd->sd_state == G_RAID_SUBDISK_S_REBUILD) {
1815                                 pd->pd_meta[j]->rebuild_lba =
1816                                     sd->sd_rebuild_pos / 512;
1817                         } else if (sd->sd_state < G_RAID_SUBDISK_S_REBUILD)
1818                                 pd->pd_meta[j]->rebuild_lba = 0;
1819                         else
1820                                 pd->pd_meta[j]->rebuild_lba = UINT32_MAX;
1821                         pd->pd_updated = 1;
1822                 }
1823         }
1824
1825         TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
1826                 pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
1827                 if (disk->d_state != G_RAID_DISK_S_ACTIVE)
1828                         continue;
1829                 if (!pd->pd_updated)
1830                         continue;
1831                 G_RAID_DEBUG(1, "Writing Promise metadata to %s",
1832                     g_raid_get_diskname(disk));
1833                 for (i = 0; i < pd->pd_subdisks; i++)
1834                         g_raid_md_promise_print(pd->pd_meta[i]);
1835                 promise_meta_write(disk->d_consumer,
1836                     pd->pd_meta, pd->pd_subdisks);
1837                 pd->pd_updated = 0;
1838         }
1839
1840         return (0);
1841 }
1842
1843 static int
1844 g_raid_md_fail_disk_promise(struct g_raid_md_object *md,
1845     struct g_raid_subdisk *tsd, struct g_raid_disk *tdisk)
1846 {
1847         struct g_raid_softc *sc;
1848         struct g_raid_md_promise_perdisk *pd;
1849         struct g_raid_subdisk *sd;
1850         int i, pos;
1851
1852         sc = md->mdo_softc;
1853         pd = (struct g_raid_md_promise_perdisk *)tdisk->d_md_data;
1854
1855         /* We can't fail disk that is not a part of array now. */
1856         if (tdisk->d_state != G_RAID_DISK_S_ACTIVE)
1857                 return (-1);
1858
1859         /*
1860          * Mark disk as failed in metadata and try to write that metadata
1861          * to the disk itself to prevent it's later resurrection as STALE.
1862          */
1863         if (pd->pd_subdisks > 0 && tdisk->d_consumer != NULL)
1864                 G_RAID_DEBUG(1, "Writing Promise metadata to %s",
1865                     g_raid_get_diskname(tdisk));
1866         for (i = 0; i < pd->pd_subdisks; i++) {
1867                 pd->pd_meta[i]->disk.flags |=
1868                     PROMISE_F_DOWN | PROMISE_F_REDIR;
1869                 pos = pd->pd_meta[i]->disk.number;
1870                 if (pos >= 0 && pos < PROMISE_MAX_DISKS) {
1871                         pd->pd_meta[i]->disks[pos].flags |=
1872                             PROMISE_F_DOWN | PROMISE_F_REDIR;
1873                 }
1874                 g_raid_md_promise_print(pd->pd_meta[i]);
1875         }
1876         if (tdisk->d_consumer != NULL)
1877                 promise_meta_write(tdisk->d_consumer,
1878                     pd->pd_meta, pd->pd_subdisks);
1879
1880         /* Change states. */
1881         g_raid_change_disk_state(tdisk, G_RAID_DISK_S_FAILED);
1882         TAILQ_FOREACH(sd, &tdisk->d_subdisks, sd_next) {
1883                 g_raid_change_subdisk_state(sd,
1884                     G_RAID_SUBDISK_S_FAILED);
1885                 g_raid_event_send(sd, G_RAID_SUBDISK_E_FAILED,
1886                     G_RAID_EVENT_SUBDISK);
1887         }
1888
1889         /* Write updated metadata to remaining disks. */
1890         g_raid_md_write_promise(md, NULL, NULL, tdisk);
1891
1892         g_raid_md_promise_refill(sc);
1893         return (0);
1894 }
1895
1896 static int
1897 g_raid_md_free_disk_promise(struct g_raid_md_object *md,
1898     struct g_raid_disk *disk)
1899 {
1900         struct g_raid_md_promise_perdisk *pd;
1901         int i;
1902
1903         pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
1904         for (i = 0; i < pd->pd_subdisks; i++) {
1905                 if (pd->pd_meta[i] != NULL) {
1906                         free(pd->pd_meta[i], M_MD_PROMISE);
1907                         pd->pd_meta[i] = NULL;
1908                 }
1909         }
1910         free(pd, M_MD_PROMISE);
1911         disk->d_md_data = NULL;
1912         return (0);
1913 }
1914
1915 static int
1916 g_raid_md_free_volume_promise(struct g_raid_md_object *md,
1917     struct g_raid_volume *vol)
1918 {
1919         struct g_raid_md_promise_pervolume *pv;
1920
1921         pv = (struct g_raid_md_promise_pervolume *)vol->v_md_data;
1922         if (pv && pv->pv_meta != NULL) {
1923                 free(pv->pv_meta, M_MD_PROMISE);
1924                 pv->pv_meta = NULL;
1925         }
1926         if (pv && !pv->pv_started) {
1927                 pv->pv_started = 1;
1928                 callout_stop(&pv->pv_start_co);
1929         }
1930         return (0);
1931 }
1932
1933 static int
1934 g_raid_md_free_promise(struct g_raid_md_object *md)
1935 {
1936
1937         return (0);
1938 }
1939
1940 G_RAID_MD_DECLARE(g_raid_md_promise);