sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_initialize.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2016 by Delphix. All rights reserved.
  24  */
  25
  26 #include <sys/spa.h>
  27 #include <sys/spa_impl.h>
  28 #include <sys/txg.h>
  29 #include <sys/vdev_impl.h>
  30 #include <sys/refcount.h>
  31 #include <sys/metaslab_impl.h>
  32 #include <sys/dsl_synctask.h>
  33 #include <sys/zap.h>
  34 #include <sys/dmu_tx.h>
  35
  36 /*
  37  * Maximum number of metaslabs per group that can be initialized
  38  * simultaneously.
  39  */
  40 int max_initialize_ms = 3;
  41
  42 /*
  43  * Value that is written to disk during initialization.
  44  */
  45 uint64_t zfs_initialize_value = 0xdeadbeefdeadbeefULL;
  46
  47 /* maximum number of I/Os outstanding per leaf vdev */
  48 int zfs_initialize_limit = 1;
  49
  50 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
  51 uint64_t zfs_initialize_chunk_size = 1024 * 1024;
  52
  53 static boolean_t
  54 vdev_initialize_should_stop(vdev_t *vd)
  55 {
  56         return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) ||
  57             vd->vdev_detached || vd->vdev_top->vdev_removing);
  58 }
  59
  60 static void
  61 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx)
  62 {
  63         /*
  64          * We pass in the guid instead of the vdev_t since the vdev may
  65          * have been freed prior to the sync task being processed. This
  66          * happens when a vdev is detached as we call spa_config_vdev_exit(),
  67          * stop the intializing thread, schedule the sync task, and free
  68          * the vdev. Later when the scheduled sync task is invoked, it would
  69          * find that the vdev has been freed.
  70          */
  71         uint64_t guid = *(uint64_t *)arg;
  72         uint64_t txg = dmu_tx_get_txg(tx);
  73         kmem_free(arg, sizeof (uint64_t));
  74
  75         vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
  76         if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
  77                 return;
  78
  79         uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK];
  80         vd->vdev_initialize_offset[txg & TXG_MASK] = 0;
  81
  82         VERIFY(vd->vdev_leaf_zap != 0);
  83
  84         objset_t *mos = vd->vdev_spa->spa_meta_objset;
  85
  86         if (last_offset > 0) {
  87                 vd->vdev_initialize_last_offset = last_offset;
  88                 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
  89                     VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
  90                     sizeof (last_offset), 1, &last_offset, tx));
  91         }
  92         if (vd->vdev_initialize_action_time > 0) {
  93                 uint64_t val = (uint64_t)vd->vdev_initialize_action_time;
  94                 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
  95                     VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val),
  96                     1, &val, tx));
  97         }
  98
  99         uint64_t initialize_state = vd->vdev_initialize_state;
 100         VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
 101             VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1,
 102             &initialize_state, tx));
 103 }
 104
 105 static void
 106 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state)
 107 {
 108         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 109         spa_t *spa = vd->vdev_spa;
 110
 111         if (new_state == vd->vdev_initialize_state)
 112                 return;
 113
 114         /*
 115          * Copy the vd's guid, this will be freed by the sync task.
 116          */
 117         uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
 118         *guid = vd->vdev_guid;
 119
 120         /*
 121          * If we're suspending, then preserving the original start time.
 122          */
 123         if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
 124                 vd->vdev_initialize_action_time = gethrestime_sec();
 125         }
 126         vd->vdev_initialize_state = new_state;
 127
 128         dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
 129         VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
 130         dsl_sync_task_nowait(spa_get_dsl(spa), vdev_initialize_zap_update_sync,
 131             guid, 2, ZFS_SPACE_CHECK_RESERVED, tx);
 132
 133         switch (new_state) {
 134         case VDEV_INITIALIZE_ACTIVE:
 135                 spa_history_log_internal(spa, "initialize", tx,
 136                     "vdev=%s activated", vd->vdev_path);
 137                 break;
 138         case VDEV_INITIALIZE_SUSPENDED:
 139                 spa_history_log_internal(spa, "initialize", tx,
 140                     "vdev=%s suspended", vd->vdev_path);
 141                 break;
 142         case VDEV_INITIALIZE_CANCELED:
 143                 spa_history_log_internal(spa, "initialize", tx,
 144                     "vdev=%s canceled", vd->vdev_path);
 145                 break;
 146         case VDEV_INITIALIZE_COMPLETE:
 147                 spa_history_log_internal(spa, "initialize", tx,
 148                     "vdev=%s complete", vd->vdev_path);
 149                 break;
 150         default:
 151                 panic("invalid state %llu", (unsigned long long)new_state);
 152         }
 153
 154         dmu_tx_commit(tx);
 155 }
 156
 157 static void
 158 vdev_initialize_cb(zio_t *zio)
 159 {
 160         vdev_t *vd = zio->io_vd;
 161         mutex_enter(&vd->vdev_initialize_io_lock);
 162         if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
 163                 /*
 164                  * The I/O failed because the vdev was unavailable; roll the
 165                  * last offset back. (This works because spa_sync waits on
 166                  * spa_txg_zio before it runs sync tasks.)
 167                  */
 168                 uint64_t *off =
 169                     &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK];
 170                 *off = MIN(*off, zio->io_offset);
 171         } else {
 172                 /*
 173                  * Since initializing is best-effort, we ignore I/O errors and
 174                  * rely on vdev_probe to determine if the errors are more
 175                  * critical.
 176                  */
 177                 if (zio->io_error != 0)
 178                         vd->vdev_stat.vs_initialize_errors++;
 179
 180                 vd->vdev_initialize_bytes_done += zio->io_orig_size;
 181         }
 182         ASSERT3U(vd->vdev_initialize_inflight, >, 0);
 183         vd->vdev_initialize_inflight--;
 184         cv_broadcast(&vd->vdev_initialize_io_cv);
 185         mutex_exit(&vd->vdev_initialize_io_lock);
 186
 187         spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
 188 }
 189
 190 /* Takes care of physical writing and limiting # of concurrent ZIOs. */
 191 static int
 192 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data)
 193 {
 194         spa_t *spa = vd->vdev_spa;
 195
 196         /* Limit inflight initializing I/Os */
 197         mutex_enter(&vd->vdev_initialize_io_lock);
 198         while (vd->vdev_initialize_inflight >= zfs_initialize_limit) {
 199                 cv_wait(&vd->vdev_initialize_io_cv,
 200                     &vd->vdev_initialize_io_lock);
 201         }
 202         vd->vdev_initialize_inflight++;
 203         mutex_exit(&vd->vdev_initialize_io_lock);
 204
 205         dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
 206         VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
 207         uint64_t txg = dmu_tx_get_txg(tx);
 208
 209         spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
 210         mutex_enter(&vd->vdev_initialize_lock);
 211
 212         if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) {
 213                 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
 214                 *guid = vd->vdev_guid;
 215
 216                 /* This is the first write of this txg. */
 217                 dsl_sync_task_nowait(spa_get_dsl(spa),
 218                     vdev_initialize_zap_update_sync, guid, 2,
 219                     ZFS_SPACE_CHECK_RESERVED, tx);
 220         }
 221
 222         /*
 223          * We know the vdev struct will still be around since all
 224          * consumers of vdev_free must stop the initialization first.
 225          */
 226         if (vdev_initialize_should_stop(vd)) {
 227                 mutex_enter(&vd->vdev_initialize_io_lock);
 228                 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
 229                 vd->vdev_initialize_inflight--;
 230                 mutex_exit(&vd->vdev_initialize_io_lock);
 231                 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
 232                 mutex_exit(&vd->vdev_initialize_lock);
 233                 dmu_tx_commit(tx);
 234                 return (SET_ERROR(EINTR));
 235         }
 236         mutex_exit(&vd->vdev_initialize_lock);
 237
 238         vd->vdev_initialize_offset[txg & TXG_MASK] = start + size;
 239         zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start,
 240             size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL,
 241             ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE));
 242         /* vdev_initialize_cb releases SCL_STATE_ALL */
 243
 244         dmu_tx_commit(tx);
 245
 246         return (0);
 247 }
 248
 249 /*
 250  * Translate a logical range to the physical range for the specified vdev_t.
 251  * This function is initially called with a leaf vdev and will walk each
 252  * parent vdev until it reaches a top-level vdev. Once the top-level is
 253  * reached the physical range is initialized and the recursive function
 254  * begins to unwind. As it unwinds it calls the parent's vdev specific
 255  * translation function to do the real conversion.
 256  */
 257 void
 258 vdev_xlate(vdev_t *vd, const range_seg_t *logical_rs, range_seg_t *physical_rs)
 259 {
 260         /*
 261          * Walk up the vdev tree
 262          */
 263         if (vd != vd->vdev_top) {
 264                 vdev_xlate(vd->vdev_parent, logical_rs, physical_rs);
 265         } else {
 266                 /*
 267                  * We've reached the top-level vdev, initialize the
 268                  * physical range to the logical range and start to
 269                  * unwind.
 270                  */
 271                 physical_rs->rs_start = logical_rs->rs_start;
 272                 physical_rs->rs_end = logical_rs->rs_end;
 273                 return;
 274         }
 275
 276         vdev_t *pvd = vd->vdev_parent;
 277         ASSERT3P(pvd, !=, NULL);
 278         ASSERT3P(pvd->vdev_ops->vdev_op_xlate, !=, NULL);
 279
 280         /*
 281          * As this recursive function unwinds, translate the logical
 282          * range into its physical components by calling the
 283          * vdev specific translate function.
 284          */
 285         range_seg_t intermediate = { 0 };
 286         pvd->vdev_ops->vdev_op_xlate(vd, physical_rs, &intermediate);
 287
 288         physical_rs->rs_start = intermediate.rs_start;
 289         physical_rs->rs_end = intermediate.rs_end;
 290 }
 291
 292 /*
 293  * Callback to fill each ABD chunk with zfs_initialize_value. len must be
 294  * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
 295  * allocation will guarantee these for us.
 296  */
 297 /* ARGSUSED */
 298 static int
 299 vdev_initialize_block_fill(void *buf, size_t len, void *unused)
 300 {
 301         ASSERT0(len % sizeof (uint64_t));
 302         for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) {
 303                 *(uint64_t *)((char *)(buf) + i) = zfs_initialize_value;
 304         }
 305         return (0);
 306 }
 307
 308 static abd_t *
 309 vdev_initialize_block_alloc()
 310 {
 311         /* Allocate ABD for filler data */
 312         abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE);
 313
 314         ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t));
 315         (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size,
 316             vdev_initialize_block_fill, NULL);
 317
 318         return (data);
 319 }
 320
 321 static void
 322 vdev_initialize_block_free(abd_t *data)
 323 {
 324         abd_free(data);
 325 }
 326
 327 static int
 328 vdev_initialize_ranges(vdev_t *vd, abd_t *data)
 329 {
 330         avl_tree_t *rt = &vd->vdev_initialize_tree->rt_root;
 331
 332         for (range_seg_t *rs = avl_first(rt); rs != NULL;
 333             rs = AVL_NEXT(rt, rs)) {
 334                 uint64_t size = rs->rs_end - rs->rs_start;
 335
 336                 /* Split range into legally-sized physical chunks */
 337                 uint64_t writes_required =
 338                     ((size - 1) / zfs_initialize_chunk_size) + 1;
 339
 340                 for (uint64_t w = 0; w < writes_required; w++) {
 341                         int error;
 342
 343                         error = vdev_initialize_write(vd,
 344                             VDEV_LABEL_START_SIZE + rs->rs_start +
 345                             (w * zfs_initialize_chunk_size),
 346                             MIN(size - (w * zfs_initialize_chunk_size),
 347                             zfs_initialize_chunk_size), data);
 348                         if (error != 0)
 349                                 return (error);
 350                 }
 351         }
 352         return (0);
 353 }
 354
 355 static void
 356 vdev_initialize_ms_load(metaslab_t *msp)
 357 {
 358         ASSERT(MUTEX_HELD(&msp->ms_lock));
 359
 360         metaslab_load_wait(msp);
 361         if (!msp->ms_loaded)
 362                 VERIFY0(metaslab_load(msp));
 363 }
 364
 365 static void
 366 vdev_initialize_mg_wait(metaslab_group_t *mg)
 367 {
 368         ASSERT(MUTEX_HELD(&mg->mg_ms_initialize_lock));
 369         while (mg->mg_initialize_updating) {
 370                 cv_wait(&mg->mg_ms_initialize_cv, &mg->mg_ms_initialize_lock);
 371         }
 372 }
 373
 374 static void
 375 vdev_initialize_mg_mark(metaslab_group_t *mg)
 376 {
 377         ASSERT(MUTEX_HELD(&mg->mg_ms_initialize_lock));
 378         ASSERT(mg->mg_initialize_updating);
 379
 380         while (mg->mg_ms_initializing >= max_initialize_ms) {
 381                 cv_wait(&mg->mg_ms_initialize_cv, &mg->mg_ms_initialize_lock);
 382         }
 383         mg->mg_ms_initializing++;
 384         ASSERT3U(mg->mg_ms_initializing, <=, max_initialize_ms);
 385 }
 386
 387 /*
 388  * Mark the metaslab as being initialized to prevent any allocations
 389  * on this metaslab. We must also track how many metaslabs are currently
 390  * being initialized within a metaslab group and limit them to prevent
 391  * allocation failures from occurring because all metaslabs are being
 392  * initialized.
 393  */
 394 static void
 395 vdev_initialize_ms_mark(metaslab_t *msp)
 396 {
 397         ASSERT(!MUTEX_HELD(&msp->ms_lock));
 398         metaslab_group_t *mg = msp->ms_group;
 399
 400         mutex_enter(&mg->mg_ms_initialize_lock);
 401
 402         /*
 403          * To keep an accurate count of how many threads are initializing
 404          * a specific metaslab group, we only allow one thread to mark
 405          * the metaslab group at a time. This ensures that the value of
 406          * ms_initializing will be accurate when we decide to mark a metaslab
 407          * group as being initialized. To do this we force all other threads
 408          * to wait till the metaslab's mg_initialize_updating flag is no
 409          * longer set.
 410          */
 411         vdev_initialize_mg_wait(mg);
 412         mg->mg_initialize_updating = B_TRUE;
 413         if (msp->ms_initializing == 0) {
 414                 vdev_initialize_mg_mark(mg);
 415         }
 416         mutex_enter(&msp->ms_lock);
 417         msp->ms_initializing++;
 418         mutex_exit(&msp->ms_lock);
 419
 420         mg->mg_initialize_updating = B_FALSE;
 421         cv_broadcast(&mg->mg_ms_initialize_cv);
 422         mutex_exit(&mg->mg_ms_initialize_lock);
 423 }
 424
 425 static void
 426 vdev_initialize_ms_unmark(metaslab_t *msp)
 427 {
 428         ASSERT(!MUTEX_HELD(&msp->ms_lock));
 429         metaslab_group_t *mg = msp->ms_group;
 430         mutex_enter(&mg->mg_ms_initialize_lock);
 431         mutex_enter(&msp->ms_lock);
 432         if (--msp->ms_initializing == 0) {
 433                 mg->mg_ms_initializing--;
 434                 cv_broadcast(&mg->mg_ms_initialize_cv);
 435         }
 436         mutex_exit(&msp->ms_lock);
 437         mutex_exit(&mg->mg_ms_initialize_lock);
 438 }
 439
 440 static void
 441 vdev_initialize_calculate_progress(vdev_t *vd)
 442 {
 443         ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
 444             spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
 445         ASSERT(vd->vdev_leaf_zap != 0);
 446
 447         vd->vdev_initialize_bytes_est = 0;
 448         vd->vdev_initialize_bytes_done = 0;
 449
 450         for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
 451                 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
 452                 mutex_enter(&msp->ms_lock);
 453
 454                 uint64_t ms_free = msp->ms_size -
 455                     space_map_allocated(msp->ms_sm);
 456
 457                 if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
 458                         ms_free /= vd->vdev_top->vdev_children;
 459
 460                 /*
 461                  * Convert the metaslab range to a physical range
 462                  * on our vdev. We use this to determine if we are
 463                  * in the middle of this metaslab range.
 464                  */
 465                 range_seg_t logical_rs, physical_rs;
 466                 logical_rs.rs_start = msp->ms_start;
 467                 logical_rs.rs_end = msp->ms_start + msp->ms_size;
 468                 vdev_xlate(vd, &logical_rs, &physical_rs);
 469
 470                 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) {
 471                         vd->vdev_initialize_bytes_est += ms_free;
 472                         mutex_exit(&msp->ms_lock);
 473                         continue;
 474                 } else if (vd->vdev_initialize_last_offset >
 475                     physical_rs.rs_end) {
 476                         vd->vdev_initialize_bytes_done += ms_free;
 477                         vd->vdev_initialize_bytes_est += ms_free;
 478                         mutex_exit(&msp->ms_lock);
 479                         continue;
 480                 }
 481
 482                 /*
 483                  * If we get here, we're in the middle of initializing this
 484                  * metaslab. Load it and walk the free tree for more accurate
 485                  * progress estimation.
 486                  */
 487                 vdev_initialize_ms_load(msp);
 488
 489                 for (range_seg_t *rs = avl_first(&msp->ms_allocatable->rt_root); rs;
 490                     rs = AVL_NEXT(&msp->ms_allocatable->rt_root, rs)) {
 491                         logical_rs.rs_start = rs->rs_start;
 492                         logical_rs.rs_end = rs->rs_end;
 493                         vdev_xlate(vd, &logical_rs, &physical_rs);
 494
 495                         uint64_t size = physical_rs.rs_end -
 496                             physical_rs.rs_start;
 497                         vd->vdev_initialize_bytes_est += size;
 498                         if (vd->vdev_initialize_last_offset >
 499                             physical_rs.rs_end) {
 500                                 vd->vdev_initialize_bytes_done += size;
 501                         } else if (vd->vdev_initialize_last_offset >
 502                             physical_rs.rs_start &&
 503                             vd->vdev_initialize_last_offset <
 504                             physical_rs.rs_end) {
 505                                 vd->vdev_initialize_bytes_done +=
 506                                     vd->vdev_initialize_last_offset -
 507                                     physical_rs.rs_start;
 508                         }
 509                 }
 510                 mutex_exit(&msp->ms_lock);
 511         }
 512 }
 513
 514 static void
 515 vdev_initialize_load(vdev_t *vd)
 516 {
 517         ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
 518             spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
 519         ASSERT(vd->vdev_leaf_zap != 0);
 520
 521         if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE ||
 522             vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) {
 523                 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
 524                     vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
 525                     sizeof (vd->vdev_initialize_last_offset), 1,
 526                     &vd->vdev_initialize_last_offset);
 527                 ASSERT(err == 0 || err == ENOENT);
 528         }
 529
 530         vdev_initialize_calculate_progress(vd);
 531 }
 532
 533
 534 /*
 535  * Convert the logical range into a physcial range and add it to our
 536  * avl tree.
 537  */
 538 void
 539 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
 540 {
 541         vdev_t *vd = arg;
 542         range_seg_t logical_rs, physical_rs;
 543         logical_rs.rs_start = start;
 544         logical_rs.rs_end = start + size;
 545
 546         ASSERT(vd->vdev_ops->vdev_op_leaf);
 547         vdev_xlate(vd, &logical_rs, &physical_rs);
 548
 549         IMPLY(vd->vdev_top == vd,
 550             logical_rs.rs_start == physical_rs.rs_start);
 551         IMPLY(vd->vdev_top == vd,
 552             logical_rs.rs_end == physical_rs.rs_end);
 553
 554         /* Only add segments that we have not visited yet */
 555         if (physical_rs.rs_end <= vd->vdev_initialize_last_offset)
 556                 return;
 557
 558         /* Pick up where we left off mid-range. */
 559         if (vd->vdev_initialize_last_offset > physical_rs.rs_start) {
 560                 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
 561                     "(%llu, %llu)", vd->vdev_path,
 562                     (u_longlong_t)physical_rs.rs_start,
 563                     (u_longlong_t)physical_rs.rs_end,
 564                     (u_longlong_t)vd->vdev_initialize_last_offset,
 565                     (u_longlong_t)physical_rs.rs_end);
 566                 ASSERT3U(physical_rs.rs_end, >,
 567                     vd->vdev_initialize_last_offset);
 568                 physical_rs.rs_start = vd->vdev_initialize_last_offset;
 569         }
 570         ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
 571
 572         /*
 573          * With raidz, it's possible that the logical range does not live on
 574          * this leaf vdev. We only add the physical range to this vdev's if it
 575          * has a length greater than 0.
 576          */
 577         if (physical_rs.rs_end > physical_rs.rs_start) {
 578                 range_tree_add(vd->vdev_initialize_tree, physical_rs.rs_start,
 579                     physical_rs.rs_end - physical_rs.rs_start);
 580         } else {
 581                 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
 582         }
 583 }
 584
 585 static void
 586 vdev_initialize_thread(void *arg)
 587 {
 588         vdev_t *vd = arg;
 589         spa_t *spa = vd->vdev_spa;
 590         int error = 0;
 591         uint64_t ms_count = 0;
 592
 593         ASSERT(vdev_is_concrete(vd));
 594         spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
 595
 596         vd->vdev_initialize_last_offset = 0;
 597         vdev_initialize_load(vd);
 598
 599         abd_t *deadbeef = vdev_initialize_block_alloc();
 600
 601         vd->vdev_initialize_tree = range_tree_create(NULL, NULL);
 602
 603         for (uint64_t i = 0; !vd->vdev_detached &&
 604             i < vd->vdev_top->vdev_ms_count; i++) {
 605                 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
 606
 607                 /*
 608                  * If we've expanded the top-level vdev or it's our
 609                  * first pass, calculate our progress.
 610                  */
 611                 if (vd->vdev_top->vdev_ms_count != ms_count) {
 612                         vdev_initialize_calculate_progress(vd);
 613                         ms_count = vd->vdev_top->vdev_ms_count;
 614                 }
 615
 616                 vdev_initialize_ms_mark(msp);
 617                 mutex_enter(&msp->ms_lock);
 618                 vdev_initialize_ms_load(msp);
 619
 620                 range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add,
 621                     vd);
 622                 mutex_exit(&msp->ms_lock);
 623
 624                 spa_config_exit(spa, SCL_CONFIG, FTAG);
 625                 error = vdev_initialize_ranges(vd, deadbeef);
 626                 vdev_initialize_ms_unmark(msp);
 627                 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
 628
 629                 range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL);
 630                 if (error != 0)
 631                         break;
 632         }
 633
 634         spa_config_exit(spa, SCL_CONFIG, FTAG);
 635         mutex_enter(&vd->vdev_initialize_io_lock);
 636         while (vd->vdev_initialize_inflight > 0) {
 637                 cv_wait(&vd->vdev_initialize_io_cv,
 638                     &vd->vdev_initialize_io_lock);
 639         }
 640         mutex_exit(&vd->vdev_initialize_io_lock);
 641
 642         range_tree_destroy(vd->vdev_initialize_tree);
 643         vdev_initialize_block_free(deadbeef);
 644         vd->vdev_initialize_tree = NULL;
 645
 646         mutex_enter(&vd->vdev_initialize_lock);
 647         if (!vd->vdev_initialize_exit_wanted && vdev_writeable(vd)) {
 648                 vdev_initialize_change_state(vd, VDEV_INITIALIZE_COMPLETE);
 649         }
 650         ASSERT(vd->vdev_initialize_thread != NULL ||
 651             vd->vdev_initialize_inflight == 0);
 652
 653         /*
 654          * Drop the vdev_initialize_lock while we sync out the
 655          * txg since it's possible that a device might be trying to
 656          * come online and must check to see if it needs to restart an
 657          * initialization. That thread will be holding the spa_config_lock
 658          * which would prevent the txg_wait_synced from completing.
 659          */
 660         mutex_exit(&vd->vdev_initialize_lock);
 661         txg_wait_synced(spa_get_dsl(spa), 0);
 662         mutex_enter(&vd->vdev_initialize_lock);
 663
 664         vd->vdev_initialize_thread = NULL;
 665         cv_broadcast(&vd->vdev_initialize_cv);
 666         mutex_exit(&vd->vdev_initialize_lock);
 667         thread_exit();
 668 }
 669
 670 /*
 671  * Initiates a device. Caller must hold vdev_initialize_lock.
 672  * Device must be a leaf and not already be initializing.
 673  */
 674 void
 675 vdev_initialize(vdev_t *vd)
 676 {
 677         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 678         ASSERT(vd->vdev_ops->vdev_op_leaf);
 679         ASSERT(vdev_is_concrete(vd));
 680         ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
 681         ASSERT(!vd->vdev_detached);
 682         ASSERT(!vd->vdev_initialize_exit_wanted);
 683         ASSERT(!vd->vdev_top->vdev_removing);
 684
 685         vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE);
 686         vd->vdev_initialize_thread = thread_create(NULL, 0,
 687             vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
 688 }
 689
 690 /*
 691  * Stop initializng a device, with the resultant initialing state being
 692  * tgt_state. Blocks until the initializing thread has exited.
 693  * Caller must hold vdev_initialize_lock and must not be writing to the spa
 694  * config, as the initializing thread may try to enter the config as a reader
 695  * before exiting.
 696  */
 697 void
 698 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state)
 699 {
 700         spa_t *spa = vd->vdev_spa;
 701         ASSERT(!spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_WRITER));
 702
 703         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 704         ASSERT(vd->vdev_ops->vdev_op_leaf);
 705         ASSERT(vdev_is_concrete(vd));
 706
 707         /*
 708          * Allow cancel requests to proceed even if the initialize thread
 709          * has stopped.
 710          */
 711         if (vd->vdev_initialize_thread == NULL &&
 712             tgt_state != VDEV_INITIALIZE_CANCELED) {
 713                 return;
 714         }
 715
 716         vdev_initialize_change_state(vd, tgt_state);
 717         vd->vdev_initialize_exit_wanted = B_TRUE;
 718         while (vd->vdev_initialize_thread != NULL)
 719                 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
 720
 721         ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
 722         vd->vdev_initialize_exit_wanted = B_FALSE;
 723 }
 724
 725 static void
 726 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state)
 727 {
 728         if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
 729                 mutex_enter(&vd->vdev_initialize_lock);
 730                 vdev_initialize_stop(vd, tgt_state);
 731                 mutex_exit(&vd->vdev_initialize_lock);
 732                 return;
 733         }
 734
 735         for (uint64_t i = 0; i < vd->vdev_children; i++) {
 736                 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state);
 737         }
 738 }
 739
 740 /*
 741  * Convenience function to stop initializing of a vdev tree and set all
 742  * initialize thread pointers to NULL.
 743  */
 744 void
 745 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state)
 746 {
 747         vdev_initialize_stop_all_impl(vd, tgt_state);
 748
 749         if (vd->vdev_spa->spa_sync_on) {
 750                 /* Make sure that our state has been synced to disk */
 751                 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
 752         }
 753 }
 754
 755 void
 756 vdev_initialize_restart(vdev_t *vd)
 757 {
 758         ASSERT(MUTEX_HELD(&spa_namespace_lock));
 759         ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
 760
 761         if (vd->vdev_leaf_zap != 0) {
 762                 mutex_enter(&vd->vdev_initialize_lock);
 763                 uint64_t initialize_state = VDEV_INITIALIZE_NONE;
 764                 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
 765                     vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE,
 766                     sizeof (initialize_state), 1, &initialize_state);
 767                 ASSERT(err == 0 || err == ENOENT);
 768                 vd->vdev_initialize_state = initialize_state;
 769
 770                 uint64_t timestamp = 0;
 771                 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
 772                     vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME,
 773                     sizeof (timestamp), 1, &timestamp);
 774                 ASSERT(err == 0 || err == ENOENT);
 775                 vd->vdev_initialize_action_time = (time_t)timestamp;
 776
 777                 if (vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
 778                     vd->vdev_offline) {
 779                         /* load progress for reporting, but don't resume */
 780                         vdev_initialize_load(vd);
 781                 } else if (vd->vdev_initialize_state ==
 782                     VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd)) {
 783                         vdev_initialize(vd);
 784                 }
 785
 786                 mutex_exit(&vd->vdev_initialize_lock);
 787         }
 788
 789         for (uint64_t i = 0; i < vd->vdev_children; i++) {
 790                 vdev_initialize_restart(vd->vdev_child[i]);
 791         }
 792 }