module/zfs/vdev_initialize.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
  24  */
  25
  26 #include <sys/spa.h>
  27 #include <sys/spa_impl.h>
  28 #include <sys/txg.h>
  29 #include <sys/vdev_impl.h>
  30 #include <sys/metaslab_impl.h>
  31 #include <sys/dsl_synctask.h>
  32 #include <sys/zap.h>
  33 #include <sys/dmu_tx.h>
  34 #include <sys/vdev_initialize.h>
  35
  36 /*
  37  * Value that is written to disk during initialization.
  38  */
  39 #ifdef _ILP32
  40 unsigned long zfs_initialize_value = 0xdeadbeefUL;
  41 #else
  42 unsigned long zfs_initialize_value = 0xdeadbeefdeadbeeeULL;
  43 #endif
  44
  45 /* maximum number of I/Os outstanding per leaf vdev */
  46 int zfs_initialize_limit = 1;
  47
  48 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
  49 unsigned long zfs_initialize_chunk_size = 1024 * 1024;
  50
  51 static boolean_t
  52 vdev_initialize_should_stop(vdev_t *vd)
  53 {
  54         return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) ||
  55             vd->vdev_detached || vd->vdev_top->vdev_removing);
  56 }
  57
  58 static void
  59 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx)
  60 {
  61         /*
  62          * We pass in the guid instead of the vdev_t since the vdev may
  63          * have been freed prior to the sync task being processed. This
  64          * happens when a vdev is detached as we call spa_config_vdev_exit(),
  65          * stop the initializing thread, schedule the sync task, and free
  66          * the vdev. Later when the scheduled sync task is invoked, it would
  67          * find that the vdev has been freed.
  68          */
  69         uint64_t guid = *(uint64_t *)arg;
  70         uint64_t txg = dmu_tx_get_txg(tx);
  71         kmem_free(arg, sizeof (uint64_t));
  72
  73         vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
  74         if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
  75                 return;
  76
  77         uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK];
  78         vd->vdev_initialize_offset[txg & TXG_MASK] = 0;
  79
  80         VERIFY(vd->vdev_leaf_zap != 0);
  81
  82         objset_t *mos = vd->vdev_spa->spa_meta_objset;
  83
  84         if (last_offset > 0) {
  85                 vd->vdev_initialize_last_offset = last_offset;
  86                 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
  87                     VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
  88                     sizeof (last_offset), 1, &last_offset, tx));
  89         }
  90         if (vd->vdev_initialize_action_time > 0) {
  91                 uint64_t val = (uint64_t)vd->vdev_initialize_action_time;
  92                 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
  93                     VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val),
  94                     1, &val, tx));
  95         }
  96
  97         uint64_t initialize_state = vd->vdev_initialize_state;
  98         VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
  99             VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1,
 100             &initialize_state, tx));
 101 }
 102
 103 static void
 104 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state)
 105 {
 106         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 107         spa_t *spa = vd->vdev_spa;
 108
 109         if (new_state == vd->vdev_initialize_state)
 110                 return;
 111
 112         /*
 113          * Copy the vd's guid, this will be freed by the sync task.
 114          */
 115         uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
 116         *guid = vd->vdev_guid;
 117
 118         /*
 119          * If we're suspending, then preserving the original start time.
 120          */
 121         if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
 122                 vd->vdev_initialize_action_time = gethrestime_sec();
 123         }
 124         vd->vdev_initialize_state = new_state;
 125
 126         dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
 127         VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
 128         dsl_sync_task_nowait(spa_get_dsl(spa), vdev_initialize_zap_update_sync,
 129             guid, 2, ZFS_SPACE_CHECK_NONE, tx);
 130
 131         switch (new_state) {
 132         case VDEV_INITIALIZE_ACTIVE:
 133                 spa_history_log_internal(spa, "initialize", tx,
 134                     "vdev=%s activated", vd->vdev_path);
 135                 break;
 136         case VDEV_INITIALIZE_SUSPENDED:
 137                 spa_history_log_internal(spa, "initialize", tx,
 138                     "vdev=%s suspended", vd->vdev_path);
 139                 break;
 140         case VDEV_INITIALIZE_CANCELED:
 141                 spa_history_log_internal(spa, "initialize", tx,
 142                     "vdev=%s canceled", vd->vdev_path);
 143                 break;
 144         case VDEV_INITIALIZE_COMPLETE:
 145                 spa_history_log_internal(spa, "initialize", tx,
 146                     "vdev=%s complete", vd->vdev_path);
 147                 break;
 148         default:
 149                 panic("invalid state %llu", (unsigned long long)new_state);
 150         }
 151
 152         dmu_tx_commit(tx);
 153
 154         if (new_state != VDEV_INITIALIZE_ACTIVE)
 155                 spa_notify_waiters(spa);
 156 }
 157
 158 static void
 159 vdev_initialize_cb(zio_t *zio)
 160 {
 161         vdev_t *vd = zio->io_vd;
 162         mutex_enter(&vd->vdev_initialize_io_lock);
 163         if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
 164                 /*
 165                  * The I/O failed because the vdev was unavailable; roll the
 166                  * last offset back. (This works because spa_sync waits on
 167                  * spa_txg_zio before it runs sync tasks.)
 168                  */
 169                 uint64_t *off =
 170                     &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK];
 171                 *off = MIN(*off, zio->io_offset);
 172         } else {
 173                 /*
 174                  * Since initializing is best-effort, we ignore I/O errors and
 175                  * rely on vdev_probe to determine if the errors are more
 176                  * critical.
 177                  */
 178                 if (zio->io_error != 0)
 179                         vd->vdev_stat.vs_initialize_errors++;
 180
 181                 vd->vdev_initialize_bytes_done += zio->io_orig_size;
 182         }
 183         ASSERT3U(vd->vdev_initialize_inflight, >, 0);
 184         vd->vdev_initialize_inflight--;
 185         cv_broadcast(&vd->vdev_initialize_io_cv);
 186         mutex_exit(&vd->vdev_initialize_io_lock);
 187
 188         spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
 189 }
 190
 191 /* Takes care of physical writing and limiting # of concurrent ZIOs. */
 192 static int
 193 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data)
 194 {
 195         spa_t *spa = vd->vdev_spa;
 196
 197         /* Limit inflight initializing I/Os */
 198         mutex_enter(&vd->vdev_initialize_io_lock);
 199         while (vd->vdev_initialize_inflight >= zfs_initialize_limit) {
 200                 cv_wait(&vd->vdev_initialize_io_cv,
 201                     &vd->vdev_initialize_io_lock);
 202         }
 203         vd->vdev_initialize_inflight++;
 204         mutex_exit(&vd->vdev_initialize_io_lock);
 205
 206         dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
 207         VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
 208         uint64_t txg = dmu_tx_get_txg(tx);
 209
 210         spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
 211         mutex_enter(&vd->vdev_initialize_lock);
 212
 213         if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) {
 214                 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
 215                 *guid = vd->vdev_guid;
 216
 217                 /* This is the first write of this txg. */
 218                 dsl_sync_task_nowait(spa_get_dsl(spa),
 219                     vdev_initialize_zap_update_sync, guid, 2,
 220                     ZFS_SPACE_CHECK_RESERVED, tx);
 221         }
 222
 223         /*
 224          * We know the vdev struct will still be around since all
 225          * consumers of vdev_free must stop the initialization first.
 226          */
 227         if (vdev_initialize_should_stop(vd)) {
 228                 mutex_enter(&vd->vdev_initialize_io_lock);
 229                 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
 230                 vd->vdev_initialize_inflight--;
 231                 mutex_exit(&vd->vdev_initialize_io_lock);
 232                 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
 233                 mutex_exit(&vd->vdev_initialize_lock);
 234                 dmu_tx_commit(tx);
 235                 return (SET_ERROR(EINTR));
 236         }
 237         mutex_exit(&vd->vdev_initialize_lock);
 238
 239         vd->vdev_initialize_offset[txg & TXG_MASK] = start + size;
 240         zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start,
 241             size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL,
 242             ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE));
 243         /* vdev_initialize_cb releases SCL_STATE_ALL */
 244
 245         dmu_tx_commit(tx);
 246
 247         return (0);
 248 }
 249
 250 /*
 251  * Callback to fill each ABD chunk with zfs_initialize_value. len must be
 252  * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
 253  * allocation will guarantee these for us.
 254  */
 255 /* ARGSUSED */
 256 static int
 257 vdev_initialize_block_fill(void *buf, size_t len, void *unused)
 258 {
 259         ASSERT0(len % sizeof (uint64_t));
 260 #ifdef _ILP32
 261         for (uint64_t i = 0; i < len; i += sizeof (uint32_t)) {
 262                 *(uint32_t *)((char *)(buf) + i) = zfs_initialize_value;
 263         }
 264 #else
 265         for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) {
 266                 *(uint64_t *)((char *)(buf) + i) = zfs_initialize_value;
 267         }
 268 #endif
 269         return (0);
 270 }
 271
 272 static abd_t *
 273 vdev_initialize_block_alloc(void)
 274 {
 275         /* Allocate ABD for filler data */
 276         abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE);
 277
 278         ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t));
 279         (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size,
 280             vdev_initialize_block_fill, NULL);
 281
 282         return (data);
 283 }
 284
 285 static void
 286 vdev_initialize_block_free(abd_t *data)
 287 {
 288         abd_free(data);
 289 }
 290
 291 static int
 292 vdev_initialize_ranges(vdev_t *vd, abd_t *data)
 293 {
 294         range_tree_t *rt = vd->vdev_initialize_tree;
 295         zfs_btree_t *bt = &rt->rt_root;
 296         zfs_btree_index_t where;
 297
 298         for (range_seg_t *rs = zfs_btree_first(bt, &where); rs != NULL;
 299             rs = zfs_btree_next(bt, &where, &where)) {
 300                 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
 301
 302                 /* Split range into legally-sized physical chunks */
 303                 uint64_t writes_required =
 304                     ((size - 1) / zfs_initialize_chunk_size) + 1;
 305
 306                 for (uint64_t w = 0; w < writes_required; w++) {
 307                         int error;
 308
 309                         error = vdev_initialize_write(vd,
 310                             VDEV_LABEL_START_SIZE + rs_get_start(rs, rt) +
 311                             (w * zfs_initialize_chunk_size),
 312                             MIN(size - (w * zfs_initialize_chunk_size),
 313                             zfs_initialize_chunk_size), data);
 314                         if (error != 0)
 315                                 return (error);
 316                 }
 317         }
 318         return (0);
 319 }
 320
 321 static void
 322 vdev_initialize_calculate_progress(vdev_t *vd)
 323 {
 324         ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
 325             spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
 326         ASSERT(vd->vdev_leaf_zap != 0);
 327
 328         vd->vdev_initialize_bytes_est = 0;
 329         vd->vdev_initialize_bytes_done = 0;
 330
 331         for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
 332                 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
 333                 mutex_enter(&msp->ms_lock);
 334
 335                 uint64_t ms_free = msp->ms_size -
 336                     metaslab_allocated_space(msp);
 337
 338                 if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
 339                         ms_free /= vd->vdev_top->vdev_children;
 340
 341                 /*
 342                  * Convert the metaslab range to a physical range
 343                  * on our vdev. We use this to determine if we are
 344                  * in the middle of this metaslab range.
 345                  */
 346                 range_seg64_t logical_rs, physical_rs;
 347                 logical_rs.rs_start = msp->ms_start;
 348                 logical_rs.rs_end = msp->ms_start + msp->ms_size;
 349                 vdev_xlate(vd, &logical_rs, &physical_rs);
 350
 351                 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) {
 352                         vd->vdev_initialize_bytes_est += ms_free;
 353                         mutex_exit(&msp->ms_lock);
 354                         continue;
 355                 } else if (vd->vdev_initialize_last_offset >
 356                     physical_rs.rs_end) {
 357                         vd->vdev_initialize_bytes_done += ms_free;
 358                         vd->vdev_initialize_bytes_est += ms_free;
 359                         mutex_exit(&msp->ms_lock);
 360                         continue;
 361                 }
 362
 363                 /*
 364                  * If we get here, we're in the middle of initializing this
 365                  * metaslab. Load it and walk the free tree for more accurate
 366                  * progress estimation.
 367                  */
 368                 VERIFY0(metaslab_load(msp));
 369
 370                 zfs_btree_index_t where;
 371                 range_tree_t *rt = msp->ms_allocatable;
 372                 for (range_seg_t *rs =
 373                     zfs_btree_first(&rt->rt_root, &where); rs;
 374                     rs = zfs_btree_next(&rt->rt_root, &where,
 375                     &where)) {
 376                         logical_rs.rs_start = rs_get_start(rs, rt);
 377                         logical_rs.rs_end = rs_get_end(rs, rt);
 378                         vdev_xlate(vd, &logical_rs, &physical_rs);
 379
 380                         uint64_t size = physical_rs.rs_end -
 381                             physical_rs.rs_start;
 382                         vd->vdev_initialize_bytes_est += size;
 383                         if (vd->vdev_initialize_last_offset >
 384                             physical_rs.rs_end) {
 385                                 vd->vdev_initialize_bytes_done += size;
 386                         } else if (vd->vdev_initialize_last_offset >
 387                             physical_rs.rs_start &&
 388                             vd->vdev_initialize_last_offset <
 389                             physical_rs.rs_end) {
 390                                 vd->vdev_initialize_bytes_done +=
 391                                     vd->vdev_initialize_last_offset -
 392                                     physical_rs.rs_start;
 393                         }
 394                 }
 395                 mutex_exit(&msp->ms_lock);
 396         }
 397 }
 398
 399 static int
 400 vdev_initialize_load(vdev_t *vd)
 401 {
 402         int err = 0;
 403         ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
 404             spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
 405         ASSERT(vd->vdev_leaf_zap != 0);
 406
 407         if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE ||
 408             vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) {
 409                 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
 410                     vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
 411                     sizeof (vd->vdev_initialize_last_offset), 1,
 412                     &vd->vdev_initialize_last_offset);
 413                 if (err == ENOENT) {
 414                         vd->vdev_initialize_last_offset = 0;
 415                         err = 0;
 416                 }
 417         }
 418
 419         vdev_initialize_calculate_progress(vd);
 420         return (err);
 421 }
 422
 423 /*
 424  * Convert the logical range into a physical range and add it to our
 425  * avl tree.
 426  */
 427 static void
 428 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
 429 {
 430         vdev_t *vd = arg;
 431         range_seg64_t logical_rs, physical_rs;
 432         logical_rs.rs_start = start;
 433         logical_rs.rs_end = start + size;
 434
 435         ASSERT(vd->vdev_ops->vdev_op_leaf);
 436         vdev_xlate(vd, &logical_rs, &physical_rs);
 437
 438         IMPLY(vd->vdev_top == vd,
 439             logical_rs.rs_start == physical_rs.rs_start);
 440         IMPLY(vd->vdev_top == vd,
 441             logical_rs.rs_end == physical_rs.rs_end);
 442
 443         /* Only add segments that we have not visited yet */
 444         if (physical_rs.rs_end <= vd->vdev_initialize_last_offset)
 445                 return;
 446
 447         /* Pick up where we left off mid-range. */
 448         if (vd->vdev_initialize_last_offset > physical_rs.rs_start) {
 449                 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
 450                     "(%llu, %llu)", vd->vdev_path,
 451                     (u_longlong_t)physical_rs.rs_start,
 452                     (u_longlong_t)physical_rs.rs_end,
 453                     (u_longlong_t)vd->vdev_initialize_last_offset,
 454                     (u_longlong_t)physical_rs.rs_end);
 455                 ASSERT3U(physical_rs.rs_end, >,
 456                     vd->vdev_initialize_last_offset);
 457                 physical_rs.rs_start = vd->vdev_initialize_last_offset;
 458         }
 459         ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
 460
 461         /*
 462          * With raidz, it's possible that the logical range does not live on
 463          * this leaf vdev. We only add the physical range to this vdev's if it
 464          * has a length greater than 0.
 465          */
 466         if (physical_rs.rs_end > physical_rs.rs_start) {
 467                 range_tree_add(vd->vdev_initialize_tree, physical_rs.rs_start,
 468                     physical_rs.rs_end - physical_rs.rs_start);
 469         } else {
 470                 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
 471         }
 472 }
 473
 474 static void
 475 vdev_initialize_thread(void *arg)
 476 {
 477         vdev_t *vd = arg;
 478         spa_t *spa = vd->vdev_spa;
 479         int error = 0;
 480         uint64_t ms_count = 0;
 481
 482         ASSERT(vdev_is_concrete(vd));
 483         spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
 484
 485         vd->vdev_initialize_last_offset = 0;
 486         VERIFY0(vdev_initialize_load(vd));
 487
 488         abd_t *deadbeef = vdev_initialize_block_alloc();
 489
 490         vd->vdev_initialize_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
 491             0, 0);
 492
 493         for (uint64_t i = 0; !vd->vdev_detached &&
 494             i < vd->vdev_top->vdev_ms_count; i++) {
 495                 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
 496                 boolean_t unload_when_done = B_FALSE;
 497
 498                 /*
 499                  * If we've expanded the top-level vdev or it's our
 500                  * first pass, calculate our progress.
 501                  */
 502                 if (vd->vdev_top->vdev_ms_count != ms_count) {
 503                         vdev_initialize_calculate_progress(vd);
 504                         ms_count = vd->vdev_top->vdev_ms_count;
 505                 }
 506
 507                 spa_config_exit(spa, SCL_CONFIG, FTAG);
 508                 metaslab_disable(msp);
 509                 mutex_enter(&msp->ms_lock);
 510                 if (!msp->ms_loaded && !msp->ms_loading)
 511                         unload_when_done = B_TRUE;
 512                 VERIFY0(metaslab_load(msp));
 513
 514                 range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add,
 515                     vd);
 516                 mutex_exit(&msp->ms_lock);
 517
 518                 error = vdev_initialize_ranges(vd, deadbeef);
 519                 metaslab_enable(msp, B_TRUE, unload_when_done);
 520                 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
 521
 522                 range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL);
 523                 if (error != 0)
 524                         break;
 525         }
 526
 527         spa_config_exit(spa, SCL_CONFIG, FTAG);
 528         mutex_enter(&vd->vdev_initialize_io_lock);
 529         while (vd->vdev_initialize_inflight > 0) {
 530                 cv_wait(&vd->vdev_initialize_io_cv,
 531                     &vd->vdev_initialize_io_lock);
 532         }
 533         mutex_exit(&vd->vdev_initialize_io_lock);
 534
 535         range_tree_destroy(vd->vdev_initialize_tree);
 536         vdev_initialize_block_free(deadbeef);
 537         vd->vdev_initialize_tree = NULL;
 538
 539         mutex_enter(&vd->vdev_initialize_lock);
 540         if (!vd->vdev_initialize_exit_wanted && vdev_writeable(vd)) {
 541                 vdev_initialize_change_state(vd, VDEV_INITIALIZE_COMPLETE);
 542         }
 543         ASSERT(vd->vdev_initialize_thread != NULL ||
 544             vd->vdev_initialize_inflight == 0);
 545
 546         /*
 547          * Drop the vdev_initialize_lock while we sync out the
 548          * txg since it's possible that a device might be trying to
 549          * come online and must check to see if it needs to restart an
 550          * initialization. That thread will be holding the spa_config_lock
 551          * which would prevent the txg_wait_synced from completing.
 552          */
 553         mutex_exit(&vd->vdev_initialize_lock);
 554         txg_wait_synced(spa_get_dsl(spa), 0);
 555         mutex_enter(&vd->vdev_initialize_lock);
 556
 557         vd->vdev_initialize_thread = NULL;
 558         cv_broadcast(&vd->vdev_initialize_cv);
 559         mutex_exit(&vd->vdev_initialize_lock);
 560
 561         thread_exit();
 562 }
 563
 564 /*
 565  * Initiates a device. Caller must hold vdev_initialize_lock.
 566  * Device must be a leaf and not already be initializing.
 567  */
 568 void
 569 vdev_initialize(vdev_t *vd)
 570 {
 571         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 572         ASSERT(vd->vdev_ops->vdev_op_leaf);
 573         ASSERT(vdev_is_concrete(vd));
 574         ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
 575         ASSERT(!vd->vdev_detached);
 576         ASSERT(!vd->vdev_initialize_exit_wanted);
 577         ASSERT(!vd->vdev_top->vdev_removing);
 578
 579         vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE);
 580         vd->vdev_initialize_thread = thread_create(NULL, 0,
 581             vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
 582 }
 583
 584 /*
 585  * Wait for the initialize thread to be terminated (cancelled or stopped).
 586  */
 587 static void
 588 vdev_initialize_stop_wait_impl(vdev_t *vd)
 589 {
 590         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 591
 592         while (vd->vdev_initialize_thread != NULL)
 593                 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
 594
 595         ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
 596         vd->vdev_initialize_exit_wanted = B_FALSE;
 597 }
 598
 599 /*
 600  * Wait for vdev initialize threads which were either to cleanly exit.
 601  */
 602 void
 603 vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list)
 604 {
 605         vdev_t *vd;
 606
 607         ASSERT(MUTEX_HELD(&spa_namespace_lock));
 608
 609         while ((vd = list_remove_head(vd_list)) != NULL) {
 610                 mutex_enter(&vd->vdev_initialize_lock);
 611                 vdev_initialize_stop_wait_impl(vd);
 612                 mutex_exit(&vd->vdev_initialize_lock);
 613         }
 614 }
 615
 616 /*
 617  * Stop initializing a device, with the resultant initializing state being
 618  * tgt_state.  For blocking behavior pass NULL for vd_list.  Otherwise, when
 619  * a list_t is provided the stopping vdev is inserted in to the list.  Callers
 620  * are then required to call vdev_initialize_stop_wait() to block for all the
 621  * initialization threads to exit.  The caller must hold vdev_initialize_lock
 622  * and must not be writing to the spa config, as the initializing thread may
 623  * try to enter the config as a reader before exiting.
 624  */
 625 void
 626 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state,
 627     list_t *vd_list)
 628 {
 629         ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
 630         ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
 631         ASSERT(vd->vdev_ops->vdev_op_leaf);
 632         ASSERT(vdev_is_concrete(vd));
 633
 634         /*
 635          * Allow cancel requests to proceed even if the initialize thread
 636          * has stopped.
 637          */
 638         if (vd->vdev_initialize_thread == NULL &&
 639             tgt_state != VDEV_INITIALIZE_CANCELED) {
 640                 return;
 641         }
 642
 643         vdev_initialize_change_state(vd, tgt_state);
 644         vd->vdev_initialize_exit_wanted = B_TRUE;
 645
 646         if (vd_list == NULL) {
 647                 vdev_initialize_stop_wait_impl(vd);
 648         } else {
 649                 ASSERT(MUTEX_HELD(&spa_namespace_lock));
 650                 list_insert_tail(vd_list, vd);
 651         }
 652 }
 653
 654 static void
 655 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state,
 656     list_t *vd_list)
 657 {
 658         if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
 659                 mutex_enter(&vd->vdev_initialize_lock);
 660                 vdev_initialize_stop(vd, tgt_state, vd_list);
 661                 mutex_exit(&vd->vdev_initialize_lock);
 662                 return;
 663         }
 664
 665         for (uint64_t i = 0; i < vd->vdev_children; i++) {
 666                 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state,
 667                     vd_list);
 668         }
 669 }
 670
 671 /*
 672  * Convenience function to stop initializing of a vdev tree and set all
 673  * initialize thread pointers to NULL.
 674  */
 675 void
 676 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state)
 677 {
 678         spa_t *spa = vd->vdev_spa;
 679         list_t vd_list;
 680
 681         ASSERT(MUTEX_HELD(&spa_namespace_lock));
 682
 683         list_create(&vd_list, sizeof (vdev_t),
 684             offsetof(vdev_t, vdev_initialize_node));
 685
 686         vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list);
 687         vdev_initialize_stop_wait(spa, &vd_list);
 688
 689         if (vd->vdev_spa->spa_sync_on) {
 690                 /* Make sure that our state has been synced to disk */
 691                 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
 692         }
 693
 694         list_destroy(&vd_list);
 695 }
 696
 697 void
 698 vdev_initialize_restart(vdev_t *vd)
 699 {
 700         ASSERT(MUTEX_HELD(&spa_namespace_lock));
 701         ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
 702
 703         if (vd->vdev_leaf_zap != 0) {
 704                 mutex_enter(&vd->vdev_initialize_lock);
 705                 uint64_t initialize_state = VDEV_INITIALIZE_NONE;
 706                 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
 707                     vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE,
 708                     sizeof (initialize_state), 1, &initialize_state);
 709                 ASSERT(err == 0 || err == ENOENT);
 710                 vd->vdev_initialize_state = initialize_state;
 711
 712                 uint64_t timestamp = 0;
 713                 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
 714                     vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME,
 715                     sizeof (timestamp), 1, &timestamp);
 716                 ASSERT(err == 0 || err == ENOENT);
 717                 vd->vdev_initialize_action_time = timestamp;
 718
 719                 if (vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
 720                     vd->vdev_offline) {
 721                         /* load progress for reporting, but don't resume */
 722                         VERIFY0(vdev_initialize_load(vd));
 723                 } else if (vd->vdev_initialize_state ==
 724                     VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) &&
 725                     !vd->vdev_top->vdev_removing &&
 726                     vd->vdev_initialize_thread == NULL) {
 727                         vdev_initialize(vd);
 728                 }
 729
 730                 mutex_exit(&vd->vdev_initialize_lock);
 731         }
 732
 733         for (uint64_t i = 0; i < vd->vdev_children; i++) {
 734                 vdev_initialize_restart(vd->vdev_child[i]);
 735         }
 736 }
 737
 738 EXPORT_SYMBOL(vdev_initialize);
 739 EXPORT_SYMBOL(vdev_initialize_stop);
 740 EXPORT_SYMBOL(vdev_initialize_stop_all);
 741 EXPORT_SYMBOL(vdev_initialize_stop_wait);
 742 EXPORT_SYMBOL(vdev_initialize_restart);
 743
 744 /* BEGIN CSTYLED */
 745 ZFS_MODULE_PARAM(zfs, zfs_, initialize_value, ULONG, ZMOD_RW,
 746         "Value written during zpool initialize");
 747
 748 ZFS_MODULE_PARAM(zfs, zfs_, initialize_chunk_size, ULONG, ZMOD_RW,
 749         "Size in bytes of writes by zpool initialize");
 750 /* END CSTYLED */